From cbc43fde3c870f6b817f21bee084e2a58662c73c Mon Sep 17 00:00:00 2001 From: Mathieu Benoit Date: Fri, 31 Oct 2025 01:10:54 -0400 Subject: [PATCH] [IMP] todo support odoo upgrade - add example odoo test - prevent delete production file with validation - add makefile with selenium - script prod to dev uninstall module after installation - adapt todo with private directory - script to download remote database - TODO show documentation for migration --- Makefile | 11 +- conf/make.docker.Makefile | 6 +- conf/make.test.Makefile | 8 +- doc/FAQ.md | 16 +- docker/docker-compose_odoo_18.0.yml | 1 + docker/docker-compose_odoo_18.0_selenium.yml | 91 + script/addons/check_addons_exist.py | 76 +- script/addons/update_prod_to_dev.sh | 14 +- .../code/git_commit_migration_addons_path.py | 113 + .../code/odoo_upgrade_code_with_dir_module.py | 98 + .../odoo_upgrade_code_with_single_module.sh | 5 + ...rade_code_with_single_module_autosearch.sh | 14 + script/config/__init__.py | 0 script/config/config_file.py | 56 + script/database/db_drop_all.py | 29 +- script/database/db_restore.py | 38 +- script/database/download_remote.py | 90 + script/database/download_remote.sh | 92 + script/database/image_db.py | 2 +- script/database/list_remote.py | 60 + script/odoo/migration/README.md | 9 + .../fix_migration_odoo140_to_odoo150.py | 24 + ...install_module_list_odoo140_to_odoo150.txt | 1 + script/open_terminal_code_generator.sh | 30 +- script/terminal/open_terminal.sh | 98 + script/terminal/validate_to_continue.sh | 27 + script/todo/README.md | 2 +- script/todo/todo.json | 8 + script/todo/todo.py | 667 ++++-- script/todo/todo_file_browser.py | 97 + script/todo/todo_upgrade.py | 2060 +++++++++++++++++ 31 files changed, 3641 insertions(+), 202 deletions(-) create mode 100644 docker/docker-compose_odoo_18.0_selenium.yml create mode 100755 script/code/git_commit_migration_addons_path.py create mode 100755 script/code/odoo_upgrade_code_with_dir_module.py create mode 100755 script/code/odoo_upgrade_code_with_single_module.sh create mode 100755 script/code/odoo_upgrade_code_with_single_module_autosearch.sh create mode 100644 script/config/__init__.py create mode 100644 script/config/config_file.py create mode 100644 script/database/download_remote.py create mode 100755 script/database/download_remote.sh create mode 100755 script/database/list_remote.py create mode 100644 script/odoo/migration/README.md create mode 100644 script/odoo/migration/fix_migration_odoo140_to_odoo150.py create mode 100644 script/odoo/migration/uninstall_module_list_odoo140_to_odoo150.txt create mode 100755 script/terminal/open_terminal.sh create mode 100755 script/terminal/validate_to_continue.sh create mode 100644 script/todo/todo_file_browser.py create mode 100755 script/todo/todo_upgrade.py diff --git a/Makefile b/Makefile index 304ef6a..57bf090 100644 --- a/Makefile +++ b/Makefile @@ -130,6 +130,10 @@ open_selenium: ############ .PHONY: format format: + ./script/maintenance/format_file_to_commit.py + +.PHONY: format_all +format_all: parallel ::: "./script/make.sh format_code_generator" "./script/make.sh format_code_generator_template" "./script/make.sh format_script" "./script/make.sh format_erplibre_addons" "./script/make.sh format_supported_addons" .PHONY: format_code_generator @@ -212,6 +216,11 @@ repo_configure_group_code_generator: repo_show_status: .venv.erplibre/bin/repo forall -pc "git status -s" +# Show git stash for all repo +.PHONY: repo_do_stash +repo_do_stash: + .venv.erplibre/bin/repo forall -pc "git stash" + # Show divergence between actual repository and production manifest .PHONY: repo_diff_manifest_production repo_diff_manifest_production: @@ -299,7 +308,7 @@ i18n_generate_demo_portal: .PHONY: clean clean: - find . -type f -name '*.py[co]' -delete -o -type d -name __pycache__ -delete + find . \( -name '__pycache__' -type d -prune -o -name '*.pyc' -o -name '*.pyo' \) -exec rm -rf {} + ############### # Statistic # diff --git a/conf/make.docker.Makefile b/conf/make.docker.Makefile index 5d308db..5b62148 100644 --- a/conf/make.docker.Makefile +++ b/conf/make.docker.Makefile @@ -107,4 +107,8 @@ docker_build_release_beta: # docker clean all .PHONY: docker_clean_all docker_clean_all: - docker system prune -a --volumes + ./script/terminal/validate_to_continue.sh "⚠️ This will REMOVE unused images, containers, networks, and VOLUMES." && docker system prune -a --volumes + +.PHONY: docker_compose_clean_all +docker_compose_clean_all: + ./script/terminal/validate_to_continue.sh "⚠️ This will REMOVE docker compose images, volumes and network." && docker-compose down --rmi all -v diff --git a/conf/make.test.Makefile b/conf/make.test.Makefile index 2e86f25..b6179ee 100644 --- a/conf/make.test.Makefile +++ b/conf/make.test.Makefile @@ -118,4 +118,10 @@ test_addons_helpdesk: ./.venv.erplibre/bin/coverage combine -a ./.venv.erplibre/bin/coverage report -m ./.venv.erplibre/bin/coverage html - ./.venv.erplibre/bin/coverage json \ No newline at end of file + ./.venv.erplibre/bin/coverage json + +.PHONY: test_addons_project_sale_link_specific_file +test_addons_project_sale_link_specific_file: + ./odoo_bin.sh db --drop --database test_addons_project_sale_link + ./script/addons/install_addons_dev.sh test_addons_project_sale_link project_sale_link + ./test.sh -d test_addons_project_sale_link --db-filter test_addons_project_sale_link -i project_sale_link --test-file=odoo18.0/addons/OCA_project/project_sale_link/tests/test_project_sale_link.py diff --git a/doc/FAQ.md b/doc/FAQ.md index a950c05..923b36d 100644 --- a/doc/FAQ.md +++ b/doc/FAQ.md @@ -1,5 +1,18 @@ # FAQ +## Into execution + +### wkthmltopdf TimeoutError + +If you find this bug on server log : +`odoo.addons.base.models.ir_actions_report: wkhtmltopdf: Exit with code 1 due to network error: TimeoutError` + +Into configuration, technique, go to ir.config_parameter (system parameter) and add configuration : +``` +key : report.url +value : http://127.0.0.1:8069 +``` + ## Scripting ### Search all duplicate file recursively in given directory @@ -188,7 +201,8 @@ limit_memory_hard = 0 ### Error non-overlapping IPv4 address pool You got this error when you start a -docker-compose: `ERROR: could not find an available, non-overlapping IPv4 address pool among the defaults to assign to the network` +docker-compose: +`ERROR: could not find an available, non-overlapping IPv4 address pool among the defaults to assign to the network` It's because the subnet is limited, you need to change it. diff --git a/docker/docker-compose_odoo_18.0.yml b/docker/docker-compose_odoo_18.0.yml index bd055f4..df70d4e 100644 --- a/docker/docker-compose_odoo_18.0.yml +++ b/docker/docker-compose_odoo_18.0.yml @@ -28,6 +28,7 @@ services: # See the volume section at the end of the file - erplibre_data_dir:/home/odoo/.local/share/Odoo - ./addons/addons:/ERPLibre/odoo18.0/addons/addons + - ./private:/ERPLibre/private - erplibre_conf:/etc/odoo restart: always diff --git a/docker/docker-compose_odoo_18.0_selenium.yml b/docker/docker-compose_odoo_18.0_selenium.yml new file mode 100644 index 0000000..9e20c4d --- /dev/null +++ b/docker/docker-compose_odoo_18.0_selenium.yml @@ -0,0 +1,91 @@ +services: + ERPLibre: + image: technolibre/erplibre:1.6.0_odoo_18.0_4f4f563 + ports: + - 8069:8069 + - 8071:8071 + - 8072:8072 + environment: + HOST: db,selenium-hub + PASSWORD: mysecretpassword + USER: odoo + POSTGRES_DB: postgres + STOP_BEFORE_INIT: "False" + DB_NAME: "" + UPDATE_ALL_DB: "False" + depends_on: + - db + - selenium-hub + # not behind a proxy + #command: odoo --workers 0 + # behind a proxy + #command: odoo --workers 2 --proxy-mode + # For production ready + #command: odoo --workers 2 --proxy-mode --no-database-list + # Increase max memory + #command: odoo --limit-memory-hard=0 --limit-memory-soft=0 + command: odoo + volumes: + # See the volume section at the end of the file + - erplibre_data_dir:/home/odoo/.local/share/Odoo + - ./addons/addons:/ERPLibre/odoo18.0/addons/addons + - ./private:/ERPLibre/private + - erplibre_conf:/etc/odoo + restart: always + + db: + image: postgis/postgis:18-3.6-alpine + environment: + POSTGRES_PASSWORD: mysecretpassword + POSTGRES_USER: odoo + POSTGRES_DB: postgres + PGDATA: /var/lib/postgresql/pgdata + volumes: + - erplibre-db-data:/var/lib/postgresql/pgdata + restart: always + + chrome: + image: selenium/node-chrome:4.34.0-20250707 + platform: linux/amd64 + shm_size: 2gb + depends_on: + - selenium-hub + environment: + - SE_EVENT_BUS_HOST=selenium-hub + + edge: + image: selenium/node-edge:4.34.0-20250707 + platform: linux/amd64 + shm_size: 2gb + depends_on: + - selenium-hub + environment: + - SE_EVENT_BUS_HOST=selenium-hub + + firefox: + image: selenium/node-firefox:4.34.0-20250707 + shm_size: 2gb + depends_on: + - selenium-hub + environment: + - SE_EVENT_BUS_HOST=selenium-hub + + # put into config.conf selenium_network_url=http://selenium-hub:4444 + # in python + # from odoo.tools import config + # config.get("selenium_network_url") + selenium-hub: + image: selenium/hub:4.34.0-20250707 + container_name: selenium-hub + ports: + - "8073:4442" + - "8074:4443" + - "8075:4444" + +# We configure volume without specific destination to let docker manage it. To configure it through docker use (read related documentation before continuing) : +# - docker volume --help +# - docker-compose down --help +volumes: + erplibre_data_dir: + erplibre_conf: + erplibre-db-data: diff --git a/script/addons/check_addons_exist.py b/script/addons/check_addons_exist.py index aa1cd0c..0e9299b 100755 --- a/script/addons/check_addons_exist.py +++ b/script/addons/check_addons_exist.py @@ -4,6 +4,7 @@ import argparse import configparser +import json import logging import os import sys @@ -52,6 +53,16 @@ def get_config(): action="store_true", help="Print path if module exist", ) + parser.add_argument( + "--output_json", + action="store_true", + help="output json for automation", + ) + parser.add_argument( + "--format_json", + action="store_true", + help="output formated json", + ) args = parser.parse_args() return args @@ -60,6 +71,8 @@ def main(): config = get_config() if config.debug: logging.getLogger().setLevel(logging.DEBUG) + if not config.output_path and config.output_json: + config.output_path = True config_parser = configparser.ConfigParser() config_parser.read(config.config) @@ -67,21 +80,30 @@ def main(): if "addons_path" in config_parser["options"]: addons_path = config_parser["options"]["addons_path"] else: - _logger.error( - "Missing item 'addons_path' in section 'options' in" - f" '{config.config}'" - ) + msg = f"Missing item 'addons_path' in section 'options' in '{config.config}'" + if not config.output_json: + _logger.error(msg) + else: + print("{'error':%s}" % msg) return -1 else: - _logger.error(f"Missing section 'options' in '{config.config}'") + msg = f"Missing section 'options' in '{config.config}'" + if not config.output_json: + _logger.error(msg) + else: + print("{'error':%s}" % msg) return -1 lst_addons_path = addons_path.strip(",").split(",") - lst_module = config.module.strip(",").split(",") + lst_module = sorted(list(set(config.module.strip(",").split(",")))) dct_module_exist = defaultdict(list) dct_module_exist_empty = defaultdict(list) lst_module_not_exist = [] + lst_error = [] + lst_exist = [] + lst_missing = [] + lst_duplicate = [] for module in lst_module: for path in lst_addons_path: @@ -102,10 +124,11 @@ def main(): is_good = False error_missing_module = True module_list = "'" + "', '".join(lst_module_not_exist) + "'" - _logger.error( - "Missing" - f" module{'s' if len(lst_module_not_exist) > 1 else ''} {module_list}" - ) + msg = f"Missing module{'s' if len(lst_module_not_exist) > 1 else ''} {module_list}" + if not config.output_json: + _logger.error(msg) + else: + lst_missing.extend(lst_module_not_exist) if dct_module_exist: for key, lst_value in dct_module_exist.items(): is_print_value = False @@ -113,20 +136,41 @@ def main(): is_print_value = True is_good = False module_list = "'" + "', '".join(lst_value) + "'" - _logger.error(f"Conflict modules: {module_list}") + msg = f"Conflict modules: {module_list}" + if not config.output_json: + _logger.error(msg) + else: + lst_duplicate.append((key, lst_value)) elif lst_value and config.output_path: is_print_value = True if is_print_value: for value in lst_value: - print(value) + if len(lst_value) == 1: + lst_exist.append((key, value)) + if not config.output_json: + print(value) if dct_module_exist_empty and not config.output_path: for key, lst_value in dct_module_exist_empty.items(): module_list = "'" + "', '".join(lst_value) + "'" - _logger.warning( - "Found this directory, but missing __manifest__.py:" - f" {module_list}" - ) + msg = f"Found this directory, but missing __manifest__.py: {module_list}" + if not config.output_json: + _logger.warning(msg) + else: + lst_error.append(msg) + + if config.output_json: + dct_json_data = { + "exist": lst_exist, + "error": lst_error, + "duplicate": lst_duplicate, + "missing": lst_missing, + } + if config.format_json: + json_data = json.dumps(dct_json_data, indent=4, sort_keys=True) + else: + json_data = json.dumps(dct_json_data) + print(json_data) if not is_good: if error_missing_module: diff --git a/script/addons/update_prod_to_dev.sh b/script/addons/update_prod_to_dev.sh index b3c3554..2c94bd4 100755 --- a/script/addons/update_prod_to_dev.sh +++ b/script/addons/update_prod_to_dev.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -Red='\033[0;31m' # Red -Color_Off='\033[0m' # Text Reset +Red='\033[0;31m' # Red +Color_Off='\033[0m' # Text Reset # This script will remove mail configuration, remove backup configuration, and force admin user to test/test echo "Update prod to dev on BD '$1'" @@ -12,3 +12,13 @@ if [[ $retVal -ne 0 ]]; then echo -e "${Red}Error${Color_Off} install_addons.sh into update_prod_to_dev.sh" exit 1 fi + +echo "Update trace of prod to dev on BD '$1'" + +./script/addons/uninstall_addons.sh "$1" user_test,disable_mail_server,disable_auto_backup + +retVal=$? +if [[ $retVal -ne 0 ]]; then + echo -e "${Red}Error${Color_Off} uninstall_addons.sh into update_prod_to_dev.sh" + exit 1 +fi diff --git a/script/code/git_commit_migration_addons_path.py b/script/code/git_commit_migration_addons_path.py new file mode 100755 index 0000000..b26849f --- /dev/null +++ b/script/code/git_commit_migration_addons_path.py @@ -0,0 +1,113 @@ +#!/usr/bin/env python3 +# © 2025 TechnoLibre (http://www.technolibre.ca) +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl) + +import argparse +import logging +import os +import subprocess + +logging.basicConfig(level=os.environ.get("LOGLEVEL", "INFO")) +_logger = logging.getLogger(__name__) + + +def get_config(): + """Parse command line arguments, extracting the config file name, + returning the union of config file and command line arguments + + :return: dict of config file settings and command line arguments + """ + # TODO update description + parser = argparse.ArgumentParser( + formatter_class=argparse.RawDescriptionHelpFormatter, + description="""\ + +""", + epilog="""\ +""", + ) + parser.add_argument( + "--path", + required=True, + help="Path addons to commit each module for migration", + ) + parser.add_argument( + "--odoo_version", + required=True, + help="The version name to update for commit, example 12.0", + ) + args = parser.parse_args() + + return args + + +def run_git_command(command, ignore_error=False): + """Exécute une commande Git et gère les erreurs.""" + result = subprocess.run( + command, capture_output=True, text=True, shell=True + ) + if result.returncode != 0 and not ignore_error: + print(f"Error executing command: {' '.join(command)}") + print(result.stderr) + return result + + +def commit_by_directory(): + """Crée un commit pour chaque répertoire contenant des fichiers modifiés.""" + + config = get_config() + + # Exécuter 'git status --porcelain' pour obtenir les fichiers modifiés + status_result = run_git_command( + f'cd "{config.path}" && git status --porcelain' + ) + if status_result.returncode != 0: + return + + # Stocker les répertoires uniques + modified_dirs = set() + for line in status_result.stdout.splitlines(): + # Extrait le chemin du fichier + file_path = line[3:].strip() + # Récupère le nom du répertoire et l'ajoute au set + dir_name = os.path.dirname(file_path) + if dir_name: # Assure que ce n'est pas le répertoire racine + modified_dirs.add(dir_name.split("/")[0]) + + # Parcourir les répertoires uniques et commiter + for directory in modified_dirs: + print(f"Processing directory: {directory}") + + # Ajoute tous les fichiers du répertoire + if os.path.exists(os.path.join(config.path, directory)): + mig_prefix_msg = "MIG" + run_git_command(f'cd "{config.path}" && git add {directory}') + + # Vérifie si le répertoire a des changements staged + commit_result_git_diff = run_git_command( + f'cd "{config.path}" && git diff --cached --quiet {directory}', + ignore_error=True, + ) + if commit_result_git_diff.returncode == 0: + print(f"No changes to commit in {directory}.") + continue + else: + mig_prefix_msg = "DEL" + run_git_command(f'cd "{config.path}" && git rm -r {directory}') + + # Crée le commit + commit_message = ( + f"[{mig_prefix_msg}] {directory}: Migration to {config.odoo_version}" + ) + commit_result = run_git_command( + f'cd "{config.path}" && git commit -m "{commit_message}"' + ) + + if commit_result.returncode == 0: + print(f"Commit successful for {directory}.") + + print("\nAll directories with changes have been processed.") + + +if __name__ == "__main__": + commit_by_directory() diff --git a/script/code/odoo_upgrade_code_with_dir_module.py b/script/code/odoo_upgrade_code_with_dir_module.py new file mode 100755 index 0000000..fd5ac32 --- /dev/null +++ b/script/code/odoo_upgrade_code_with_dir_module.py @@ -0,0 +1,98 @@ +#!/usr/bin/env python3 +# © 2025 TechnoLibre (http://www.technolibre.ca) +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl) + +import argparse +import asyncio +import asyncio.subprocess as asp +import logging +import os +import sys +from pathlib import Path + +logging.basicConfig(level=os.environ.get("LOGLEVEL", "INFO")) +_logger = logging.getLogger(__name__) + +# TODO this script need odoo 18 + + +def get_config(): + """Parse command line arguments, extracting the config file name, + returning the union of config file and command line arguments + + :return: dict of config file settings and command line arguments + """ + # TODO update description + parser = argparse.ArgumentParser( + formatter_class=argparse.RawDescriptionHelpFormatter, + description="""\ + +""", + epilog="""\ +""", + ) + parser.add_argument( + "--path", required=True, help="Path to directory to migrate all module" + ) + args = parser.parse_args() + + return args + + +async def run_cmd(cmd: str): + """Run a command asynchronously and log output.""" + proc = await asp.create_subprocess_shell( + cmd, + stdout=asp.PIPE, + stderr=asp.PIPE, + ) + stdout, stderr = await proc.communicate() + + if stdout: + _logger.info(stdout.decode().strip()) + if stderr: + _logger.error(stderr.decode().strip()) + + return proc.returncode + + +async def migrate_modules(config): + """Find Odoo modules and run migration script on each.""" + chemin_du_dossier = Path(config.path) + if not chemin_du_dossier.exists(): + die(True, f"Path {chemin_du_dossier} does not exist") + + tasks = [] + for element in chemin_du_dossier.iterdir(): + manifest = element / "__manifest__.py" + if element.is_dir() and manifest.exists(): + cmd = f"./script/code/odoo_upgrade_code_with_single_module_autosearch.sh {element.name}" + _logger.info("Executing: %s", cmd) + tasks.append(run_cmd(cmd)) + + if tasks: + results = await asyncio.gather(*tasks, return_exceptions=True) + for idx, res in enumerate(results): + if isinstance(res, Exception): + _logger.error("Task %d failed with exception: %s", idx, res) + elif res != 0: + _logger.error("Task %d exited with code %d", idx, res) + else: + _logger.info("Task %d completed successfully", idx) + else: + _logger.warning("No modules found in %s", chemin_du_dossier) + + +def die(cond, message, code=1): + if cond: + print(message, file=sys.stderr) + sys.exit(code) + + +async def async_main(): + config = get_config() + await migrate_modules(config) + + +if __name__ == "__main__": + asyncio.run(async_main()) diff --git a/script/code/odoo_upgrade_code_with_single_module.sh b/script/code/odoo_upgrade_code_with_single_module.sh new file mode 100755 index 0000000..0fb1adc --- /dev/null +++ b/script/code/odoo_upgrade_code_with_single_module.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +# you need odoo 18 installed +# you need $1 path to the module +./odoo18.0/odoo/odoo-bin upgrade_code --from 12.0 --addons-path "$1" diff --git a/script/code/odoo_upgrade_code_with_single_module_autosearch.sh b/script/code/odoo_upgrade_code_with_single_module_autosearch.sh new file mode 100755 index 0000000..f94bfb2 --- /dev/null +++ b/script/code/odoo_upgrade_code_with_single_module_autosearch.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +Red='\033[0;31m' # Red +Color_Off='\033[0m' # Text Reset + +# you need odoo 18 installed +# you need $1 for the module name +output=$(./script/addons/check_addons_exist.py --output_path -m "$1") +retVal=$? +if [[ $retVal -ne 0 ]]; then + echo -e "${Red}Error${Color_Off} check_addons_exist.py into odoo_upgrade_code_with_single_module_autosearch.sh" + exit 1 +fi + +./script/code/odoo_upgrade_code_with_single_module.sh "$output" diff --git a/script/config/__init__.py b/script/config/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/script/config/config_file.py b/script/config/config_file.py new file mode 100644 index 0000000..878ea4a --- /dev/null +++ b/script/config/config_file.py @@ -0,0 +1,56 @@ +#!/usr/bin/env python3 +# © 2021-2025 TechnoLibre (http://www.technolibre.ca) +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl) + +import json +import logging +import os + +CONFIG_FILE = "./script/todo/todo.json" +CONFIG_OVERRIDE_FILE = "./private/todo/todo_override.json" +CONFIG_OVERRIDE_PRIVATE_FILE = "./private/todo/todo_override_private.json" +LOGO_ASCII_FILE = "./script/todo/logo_ascii.txt" + +logging.basicConfig( + format=( + "%(asctime)s,%(msecs)d %(levelname)-8s [%(filename)s:%(lineno)d]" + " %(message)s" + ), + datefmt="%Y-%m-%d:%H:%M:%S", + level=logging.INFO, +) +_logger = logging.getLogger(__name__) + + +class ConfigFile: + def get_config(self, lst_params): + # Open file + config_file = CONFIG_FILE + if os.path.exists(CONFIG_OVERRIDE_FILE): + config_file = CONFIG_OVERRIDE_FILE + + find_in_private = False + if os.path.exists(CONFIG_OVERRIDE_PRIVATE_FILE): + with open(CONFIG_OVERRIDE_PRIVATE_FILE) as cfg: + dct_data = json.load(cfg) + for param in lst_params: + if param in dct_data.keys(): + find_in_private = True + dct_data = dct_data[param] + + if not find_in_private: + with open(config_file) as cfg: + dct_data = json.load(cfg) + for param in lst_params: + try: + dct_data = dct_data[param] + except KeyError: + _logger.error( + f"KeyError on file {config_file} with keys" + f" {lst_params}" + ) + return {} + return dct_data + + def get_logo_ascii_file_path(self): + return LOGO_ASCII_FILE diff --git a/script/database/db_drop_all.py b/script/database/db_drop_all.py index 0121449..a72066c 100755 --- a/script/database/db_drop_all.py +++ b/script/database/db_drop_all.py @@ -30,6 +30,10 @@ def get_config(): help="test and test_* and other by system test.", action="store_true", ) + parser.add_argument( + "--database", + help="Specify database to delete, separate by coma", + ) args = parser.parse_args() return args @@ -39,6 +43,16 @@ def main(): out_db = execute_shell("./odoo_bin.sh db --list") lst_db = out_db.split("\n") + + lst_database_to_delete = [] + if config.database: + lst_database_to_delete = [ + a.strip() for a in config.database.split(",") + ] + + cmd_all = "parallel :::" + cmd_end = "" + lst_db_name = [] for db_name in lst_db: if config.test_only and not ( db_name in ("test",) @@ -46,11 +60,16 @@ def main(): or db_name.startswith("new_project_") ): continue - execute_shell( - "./odoo_bin.sh db --drop --database" - f" {db_name}" - ) - print(f"{db_name} deleted") + if lst_database_to_delete and db_name not in lst_database_to_delete: + continue + + cmd_end += f' "./odoo_bin.sh db --drop --database {db_name}"' + lst_db_name.append(db_name) + if cmd_end: + execute_shell(cmd_all + cmd_end) + print("Database deleted :") + for db_name in lst_db_name: + print(db_name) return 0 diff --git a/script/database/db_restore.py b/script/database/db_restore.py index 2438cc6..a1629e8 100755 --- a/script/database/db_restore.py +++ b/script/database/db_restore.py @@ -50,6 +50,16 @@ SUGGESTION action="store_true", help="Delete all database cache to clone, begin by _cache_.", ) + parser.add_argument( + "--ignore_cache", + action="store_true", + help="Ignore creating _cache_ when restoring.", + ) + parser.add_argument( + "--only_drop", + action="store_true", + help="Will only drop database if exist.", + ) args = parser.parse_args() return args @@ -122,8 +132,10 @@ def main(): arg = f"{arg_base} --drop --database {config.database}" out = check_output(arg.split(" ")).decode() print(out) + if config.only_drop: + return # Check cache exist - if cache_database not in lst_db_cache: + if cache_database not in lst_db_cache and not config.ignore_cache: _logger.info( f"## Create cache {cache_database} from image" f" {config.image} ##" @@ -135,13 +147,23 @@ def main(): out = check_output(arg.split(" ")).decode() print(out) # Clone database - _logger.info( - f"## Clone cache {cache_database} to database {config.database} ##" - ) - arg = ( - f"{arg_base} --clone --from_database" - f" {cache_database} --database {config.database}" - ) + if config.ignore_cache: + _logger.info( + f"## Restoring {config.image} to database {config.database} ##" + ) + arg = ( + f"{arg_base} --restore --restore_image" + f" {config.image} --database {config.database}" + ) + else: + _logger.info( + f"## Clone cache {cache_database} to database {config.database} ##" + ) + arg = ( + f"{arg_base} --clone --from_database" + f" {cache_database} --database {config.database}" + ) + print(arg) out = check_output(arg.split(" ")).decode() print(out) diff --git a/script/database/download_remote.py b/script/database/download_remote.py new file mode 100644 index 0000000..5e6f239 --- /dev/null +++ b/script/database/download_remote.py @@ -0,0 +1,90 @@ +#!/usr/bin/env python3 +# © 2021-2024 TechnoLibre (http://www.technolibre.ca) +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl) + +print("This script only work with localhost:8069, not working with remote instance.") + +import requests +import sys +import logging +from pathlib import Path + +ODOO_URL = '' # Your Odoo server URL +DATABASE_NAME = '' +MASTER_PASSWORD = '' +# BACKUP_FORMAT = env('BACKUP_FORMAT', default='zip') # 'zip' or 'dump' +BACKUP_FORMAT = 'zip' # 'zip' or 'dump' +OUTPUT_FILE_NAME = f'{DATABASE_NAME}_backup.{BACKUP_FORMAT}' + +# --- Logger Setup --- +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + +# TODO not working with remote, only localhost +logger.error("This script is not working with remote instance, only http://127.0.0.1:8069") + +# --- Function to download the database --- +def download_odoo_db(): + logger.info(f"Attempting to download database '{DATABASE_NAME}' from URL '{ODOO_URL}'...") + + # URL for the backup endpoint + backup_url = f'{ODOO_URL}/web/database/backup' + + # Form data for the POST request + payload = { + 'master_pwd': MASTER_PASSWORD, + 'name': DATABASE_NAME, + 'backup_format': BACKUP_FORMAT, + } + + try: + # Execute the POST request to start the backup + response = requests.post(backup_url, data=payload, stream=True) + response.raise_for_status() # Raise an exception for bad status codes + + # --- VALIDATION --- + content_type = response.headers.get('Content-Type', '').split(';')[0] + + # Check if the content type is valid + expected_types = ['application/zip', 'application/octet-stream'] + if content_type not in expected_types: + logger.error(f"ERROR: Expected one of {expected_types} but got {content_type}.") + logger.error("This usually indicates a server-side error or an incorrect master password.") + sys.exit(1) + + # Check if the content is an HTML page (to handle incorrect passwords) + first_chunk = next(response.iter_content(chunk_size=128), b'') + if first_chunk.startswith(b'<'): + logger.error("ERROR: It seems the server returned an HTML page instead of a database file.") + logger.error("This is often due to an incorrect master password or an invalid request.") + logger.error("Server Response (First 200 chars):") + logger.error(response.text[:200]) + sys.exit(1) + + # --- DOWNLOAD --- + logger.info(f"Download started, saving to '{OUTPUT_FILE_NAME}'...") + + output_path = Path(OUTPUT_FILE_NAME) + with open(output_path, 'wb') as f: + # Write the content from the first chunk to the file + f.write(first_chunk) + # Continue writing the rest of the stream in chunks + for chunk in response.iter_content(chunk_size=8192): + f.write(chunk) + + logger.info(f"Download successful! File saved at: {output_path.resolve()}") + + except requests.exceptions.RequestException as e: + logger.error(f"An error occurred while connecting to the Odoo server: {e}") + except Exception as e: + logger.error(f"An unexpected error occurred: {e}") + + +# --- Script execution --- +if __name__ == '__main__': + # Add a simple check to ensure the user has configured the variables + if ODOO_URL == 'http://localhost:8069' and DATABASE_NAME == 'your_database_name': + logger.error("Please configure the ODOO_URL, DATABASE_NAME, and MASTER_PASSWORD variables at the start of the script.") + sys.exit(1) + + download_odoo_db() diff --git a/script/database/download_remote.sh b/script/database/download_remote.sh new file mode 100755 index 0000000..652647c --- /dev/null +++ b/script/database/download_remote.sh @@ -0,0 +1,92 @@ +#!/usr/bin/env bash + +# --- Configuration --- +# Set variable +# export MASTER_PWD="" +# export DATABASE_NAME="" +# export OUTPUT_FILE_PATH="" +# export ODOO_URL="" + +#MASTER_PWD="ADMIN" +#DATABASE_NAME="BD" +BACKUP_FORMAT="zip" +#OUTPUT_FILE_PATH="/tmp/test.zip" +#ODOO_URL="https://mondomain" + +QUIET_MODE=false + +for arg in "$@"; do + case "$arg" in + --quiet) + QUIET_MODE=true + ;; + *) + echo "Argument non reconnu : $arg" + exit 1 + ;; + esac +done + +# --- Security Check --- +# Check if the MASTER_PWD environment variable is set +if [[ -z "$MASTER_PWD" ]]; then + echo "Error: The MASTER_PWD environment variable is not set." >&2 + echo "Please set it before running this script:" >&2 + echo " export MASTER_PWD='your_master_password'" >&2 + exit 1 +fi +# Check if the ODOO_URL environment variable is set +if [[ -z "$ODOO_URL" ]]; then + # echo "Error: The ODOO_URL environment variable is not set." >&2 + # echo "Please set it before running this script:" >&2 + # echo " export ODOO_URL='your_master_password'" >&2 + # exit 1 + read -p "Odoo URL: " ODOO_URL +fi +# Check if the DATABASE_NAME environment variable is set +if [[ -z "$DATABASE_NAME" ]]; then + # echo "Error: The DATABASE_NAME environment variable is not set." >&2 + # echo "Please set it before running this script:" >&2 + # echo " export DATABASE_NAME='your_master_password'" >&2 + # exit 1 + read -p "Database: " DATABASE_NAME +fi +# Check if the OUTPUT_FILE_PATH environment variable is set +if [[ -z "$OUTPUT_FILE_PATH" ]]; then + # echo "Error: The OUTPUT_FILE_PATH environment variable is not set." >&2 + # echo "Please set it before running this script:" >&2 + # echo " export OUTPUT_FILE_PATH='your_master_password'" >&2 + # exit 1 + read -p "Output File Path: " OUTPUT_FILE_PATH +fi + +ODOO_BACKUP_URL="${ODOO_URL}/web/database/backup" + +# --- Curl Command to Download Database --- +echo "Starting Odoo database backup for '${DATABASE_NAME}' from '${ODOO_BACKUP_URL}' to path '${OUTPUT_FILE_PATH}'..." + +if [ "$QUIET_MODE" = false ]; then + curl -X POST \ + -F "master_pwd=$MASTER_PWD" \ + -F "name=$DATABASE_NAME" \ + -F "backup_format=$BACKUP_FORMAT" \ + -o "$OUTPUT_FILE_PATH" \ + --progress-bar \ + "$ODOO_BACKUP_URL" +else + curl -X POST \ + -F "master_pwd=$MASTER_PWD" \ + -F "name=$DATABASE_NAME" \ + -F "backup_format=$BACKUP_FORMAT" \ + -o "$OUTPUT_FILE_PATH" \ + "$ODOO_BACKUP_URL" +fi + +# --- Verification --- +if [[ $? -eq 0 ]]; then + echo "Backup completed successfully!" + echo "File saved to: $OUTPUT_FILE_PATH" +else + echo "Backup failed. Please check the logs." >&2 + exit 1 +fi diff --git a/script/database/image_db.py b/script/database/image_db.py index d205239..f519a0e 100755 --- a/script/database/image_db.py +++ b/script/database/image_db.py @@ -100,7 +100,7 @@ def main(): if not os.path.isfile(filename_odoo_version): _logger.error(f"Missing file {filename_odoo_version}") sys.exit(1) - with open(".odoo-version", "r") as f: + with open(filename_odoo_version, "r") as f: config.odoo_version = f.readline() # Open configuration file diff --git a/script/database/list_remote.py b/script/database/list_remote.py new file mode 100755 index 0000000..d224c30 --- /dev/null +++ b/script/database/list_remote.py @@ -0,0 +1,60 @@ +#!/usr/bin/env bash + +import xmlrpc.client +import sys +import click + + +def get_db_list_xmlrpc(odoo_url): + """ + Retrieves the list of Odoo databases using the XML-RPC API. + """ + try: + common = xmlrpc.client.ServerProxy(f'{odoo_url}/xmlrpc/db') + db_list = common.list() + return db_list + except xmlrpc.client.Fault as e: + print(f"XML-RPC Error: {e.faultCode} - {e.faultString}", file=sys.stderr) + return [] + except Exception as e: + print(f"Connection Error: {e}", file=sys.stderr) + return [] + +# --- CLI using Click --- +@click.command() +@click.option( + '--odoo-url', + default='http://localhost:8069', + help='URL of the Odoo server.', + show_default=True +) +@click.option( + '--raw', + is_flag=True, + help='Output one database per line, without extra formatting. Useful for scripting.' +) +def list_databases(odoo_url, raw=False): + """ + This script lists all available databases on an Odoo server. + """ + if not raw: + click.echo(f"Attempting to connect to Odoo at: {odoo_url}") + + databases = get_db_list_xmlrpc(odoo_url) + + if databases: + if not raw: + click.echo("\nAvailable databases:") + for db in databases: + if not raw: + click.echo(f"- {db}") + else: + click.echo(db) + else: + if not raw: + click.echo("Failed to retrieve the database list.") + + +# --- Script Execution --- +if __name__ == '__main__': + list_databases() diff --git a/script/odoo/migration/README.md b/script/odoo/migration/README.md new file mode 100644 index 0000000..a5e395b --- /dev/null +++ b/script/odoo/migration/README.md @@ -0,0 +1,9 @@ +# Migration + +Run this script when doing database migration. Example : + +```bash +cat ./script/migration/fix_migration_odoo140_to_odoo150.py | ./odoo15.0/odoo/odoo-bin shell -d DATABASE +``` + +Check [uninstall_module_list_odoo140_to_odoo150.txt](uninstall_module_list_odoo140_to_odoo150.txt) diff --git a/script/odoo/migration/fix_migration_odoo140_to_odoo150.py b/script/odoo/migration/fix_migration_odoo140_to_odoo150.py new file mode 100644 index 0000000..21821eb --- /dev/null +++ b/script/odoo/migration/fix_migration_odoo140_to_odoo150.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python3 +# © 2021-2025 TechnoLibre (http://www.technolibre.ca) +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl) + +# This script need to be run when upgrade 14 to 15 when database is created from ERPLibre. +print("Running a script in the Odoo shell!") + +i = 0 +print( + f"{i}. Remove project_type because OCA project_category migrating to OCA project_type and conflict with odoo14.0/addons/Numigi_odoo-project-addons" +) +# TODO do a migration, copy data to another temporary table +env["ir.module.module"].search([("name", "=", "project_type")]).unlink() +i += 1 + +# print( +# f"{i}. Remove project_type because OCA project_category migrating to OCA project_type" +# ) +# env["ir.module.module"].search([("name", "=", "l10n_eu_oss_oca")]).unlink() +# i += 1 + +env.cr.commit() + +print("End fix migration Odoo 14.0 to Odoo 15.0") diff --git a/script/odoo/migration/uninstall_module_list_odoo140_to_odoo150.txt b/script/odoo/migration/uninstall_module_list_odoo140_to_odoo150.txt new file mode 100644 index 0000000..5917580 --- /dev/null +++ b/script/odoo/migration/uninstall_module_list_odoo140_to_odoo150.txt @@ -0,0 +1 @@ +muk_web_theme diff --git a/script/open_terminal_code_generator.sh b/script/open_terminal_code_generator.sh index 195fac8..a67943f 100755 --- a/script/open_terminal_code_generator.sh +++ b/script/open_terminal_code_generator.sh @@ -2,13 +2,27 @@ # Open a new gnome-terminal with different path on new tab ODOO_VERSION=$(cat .odoo-version) working_path=$(readlink -f .) + +# TODO fix open_terminal.sh +#paths="${working_path}/ +#${working_path}/ +#${working_path}/odoo${ODOO_VERSION}/addons/ERPLibre_erplibre_addons +#${working_path}/odoo${ODOO_VERSION}/addons/TechnoLibre_odoo-code-generator +#${working_path}/odoo${ODOO_VERSION}/addons/TechnoLibre_odoo-code-generator-template" +# +## "${working_path}/odoo${ODOO_VERSION}/addons/OCA_server-tools" +# +#cmd="git status" +##echo "${paths}" +#./script/terminal/open_terminal.sh "$cmd" "$paths" + paths=( "${working_path}/" "${working_path}/" - "${working_path}/addons.odoo${ODOO_VERSION}/ERPLibre_erplibre_addons" - "${working_path}/addons.odoo${ODOO_VERSION}/TechnoLibre_odoo-code-generator" - "${working_path}/addons.odoo${ODOO_VERSION}/TechnoLibre_odoo-code-generator-template" -# "${working_path}/addons.odoo${ODOO_VERSION}/OCA_server-tools" + "${working_path}/odoo${ODOO_VERSION}/addons/ERPLibre_erplibre_addons" + "${working_path}/odoo${ODOO_VERSION}/addons/TechnoLibre_odoo-code-generator" + "${working_path}/odoo${ODOO_VERSION}/addons/TechnoLibre_odoo-code-generator-template" +# "${working_path}/odoo${ODOO_VERSION}/addons/OCA_server-tools" ) @@ -16,7 +30,7 @@ first_iteration=true second_iteration=true if [[ "${OSTYPE}" == "linux-gnu" ]]; then cmd_before="cd " - cmd_after_first=";gnome-terminal --tab -- bash -c 'source ./.venv/bin/activate;git status;bash';" + cmd_after_first=";gnome-terminal --tab -- bash -c 'source ./.venv.erplibre/bin/activate;git status;bash';" cmd_after=";gnome-terminal --tab -- bash -c 'git status;bash';" LONGCMD="" for t in "${paths[@]}"; do @@ -43,17 +57,17 @@ elif [[ "${OSTYPE}" == "darwin"* ]]; then # Boucle pour ajouter des commandes pour ouvrir de nouveaux onglets et exécuter les scripts batch for t in "${paths[@]}"; do if $first_iteration; then - osascript_command+=" -e 'tell application \"System Events\" to keystroke \"t\" using {command down}' -e 'delay 0.1' -e 'do script \"cd ${t}; source ./.venv/bin/activate; git status\" in front window'" + osascript_command+=" -e 'tell application \"System Events\" to keystroke \"t\" using {command down}' -e 'delay 0.1' -e 'do script \"cd ${t}; source ./.venv.erplibre/bin/activate; git status\" in front window'" first_iteration=false elif $second_iteration; then - osascript_command+=" -e 'tell application \"System Events\" to keystroke \"t\" using {command down}' -e 'delay 0.1' -e 'do script \"cd ${t}; source ./.venv/bin/activate; git status\" in front window'" + osascript_command+=" -e 'tell application \"System Events\" to keystroke \"t\" using {command down}' -e 'delay 0.1' -e 'do script \"cd ${t}; source ./.venv.erplibre/bin/activate; git status\" in front window'" second_iteration=false else osascript_command+=" -e 'tell application \"System Events\" to keystroke \"t\" using {command down}' -e 'delay 0.1' -e 'do script \"cd ${t}; git status\" in front window'" fi done osascript_command+=" -e 'end tell'" - + echo "$osascript_command" # Exécution de la commande osascript eval "$osascript_command" fi diff --git a/script/terminal/open_terminal.sh b/script/terminal/open_terminal.sh new file mode 100755 index 0000000..7a44439 --- /dev/null +++ b/script/terminal/open_terminal.sh @@ -0,0 +1,98 @@ +#!/usr/bin/env bash + +CMD_TO_EXEC="$1" +PATHS_RAW="$2" +#paths="$2" +echo "Script start open_terminal.sh" +echo "$PATHS_RAW" +IFS=$'\n' read -r -d '' -a paths <<< "$PATHS_RAW" +echo "$path" +SOURCE_CMD="source ./.venv.erplibre/bin/activate" +FIRST_ITERATION=true +SECOND_ITERATION=true + +which gnome-terminal +GNOME_TERMINAL_CMD=$(which gnome-terminal) + +HAS_GNOME_TERMINAL=false +HAS_TELL_TERMINAL=false +if command -v gnome-terminal &>/dev/null; then + HAS_GNOME_TERMINAL=true + echo "Detect gnome-terminal" + +##elif command -v tell &>/dev/null; then +#elif [[ "${OSTYPE}" == "darwin"* ]] && command -v osascript &>/dev/null; then +#if [[ "${OSTYPE}" == "darwin"* ]] && command -v osascript &>/dev/null; then +# TODO validate osascript is implemented +elif [[ "${OSTYPE}" == "darwin"* ]] then + echo "Detect osascript Darwin" + HAS_TELL_TERMINAL=true +else + echo "Detect cli" +fi + +if [[ -z "$paths" ]]; then + working_path=$(readlink -f .) + paths="${working_path}/" +fi + +#if [[ "${OSTYPE}" == "linux-gnu" ]]; then +if [ "$HAS_GNOME_TERMINAL" = true ]; then + CMD_BEFORE="cd " + CMD_AFTER_FIRST=";gnome-terminal --tab -- /bin/bash -c '${SOURCE_CMD};${CMD_TO_EXEC};bash';" + CMD_AFTER=";gnome-terminal --tab -- /bin/bash -c '${CMD_TO_EXEC};bash';" + LONGCMD="" + for PATH in "${paths[@]}"; do + if [[ ! -e "${PATH}" ]]; then + continue + fi + if $FIRST_ITERATION; then + LONGCMD+="${CMD_BEFORE}${PATH}${CMD_AFTER_FIRST}" + FIRST_ITERATION=false + else + LONGCMD+="${CMD_BEFORE}${PATH}${CMD_AFTER}" + fi + done + echo "${LONGCMD}" + echo "${GNOME_TERMINAL_CMD}" +# $GNOME_TERMINAL_CMD --window -- /bin/bash -c "ls" + $GNOME_TERMINAL_CMD --window -- /bin/bash -c "${LONGCMD}" +elif [ "$HAS_TELL_TERMINAL" = true ]; then + paths=("${paths[@]:1}") + + # Initialisation de la commande osascript + osascript_command="osascript -e 'tell application \"Terminal\"'" + + # Boucle pour ajouter des commandes pour ouvrir de nouveaux onglets et exécuter les scripts batch + for PATH in "${paths[@]}"; do + if [[ ! -e "${PATH}" ]]; then + continue + fi + if $FIRST_ITERATION; then + osascript_command+=" -e 'tell application \"System Events\" to keystroke \"t\" using {command down}' -e 'delay 0.1' -e 'do script \"cd ${PATH}; ${SOURCE_CMD}; ${CMD_TO_EXEC}\" in front window'" + FIRST_ITERATION=false + elif $SECOND_ITERATION; then + osascript_command+=" -e 'tell application \"System Events\" to keystroke \"t\" using {command down}' -e 'delay 0.1' -e 'do script \"cd ${PATH}; ${SOURCE_CMD}; ${CMD_TO_EXEC}\" in front window'" + SECOND_ITERATION=false + else + osascript_command+=" -e 'tell application \"System Events\" to keystroke \"t\" using {command down}' -e 'delay 0.1' -e 'do script \"cd ${PATH}; ${SOURCE_CMD}; ${CMD_TO_EXEC}\" in front window'" + fi + done + osascript_command+=" -e 'end tell'" + + # Exécution de la commande osascript + echo "${osascript_command}" + eval "$osascript_command" +else +# echo "CLI" +# for PATH in "${paths[@]}"; do +# if [[ ! -e "${PATH}" ]]; then +# continue +# fi +# cd ${PATH} +# echo "${CMD_TO_EXEC}" +# eval ${CMD_TO_EXEC} +# done + echo "Cannot find gnome-terminal (GNOME) or osasscript (OSX)" + exit 1 +fi diff --git a/script/terminal/validate_to_continue.sh b/script/terminal/validate_to_continue.sh new file mode 100755 index 0000000..0930d27 --- /dev/null +++ b/script/terminal/validate_to_continue.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ $# -lt 1 ]]; then + echo "Usage: $0 \"Warning message to display\"" >&2 + exit 2 +fi + +msg="$1" + +# Require an interactive terminal +if [[ ! -t 0 || ! -t 1 ]]; then + echo "Interactive confirmation not possible (no TTY). Aborting." >&2 + exit 130 +fi + +echo "$msg" +printf "Proceed? [y/N] " +# shellcheck disable=SC2162 +read -r ans + +if [[ "$ans" =~ ^[yY]$ ]]; then + exit 0 +else + echo "Aborted." + exit 130 +fi diff --git a/script/todo/README.md b/script/todo/README.md index 245167c..168bde7 100644 --- a/script/todo/README.md +++ b/script/todo/README.md @@ -1,4 +1,4 @@ TODO is an assistant robot to use ERPLibre Execute it with `./script/todo/todo.py` or `make todo`. -For a new project, copy todo_example.json to private/todo.json and edit it. +For a new project, copy todo_example.json to private/todo/todo_override.json | private/todo/todo_override_private.json and edit it. diff --git a/script/todo/todo.json b/script/todo/todo.json index 7f74333..07157dd 100644 --- a/script/todo/todo.json +++ b/script/todo/todo.json @@ -32,6 +32,14 @@ { "prompt_description": "Show code status", "makefile_cmd": "repo_show_status" + }, + { + "prompt_description": "Stash all code", + "makefile_cmd": "repo_do_stash" + }, + { + "prompt_description": "Format modified code", + "makefile_cmd": "format" } ] } diff --git a/script/todo/todo.py b/script/todo/todo.py index 8c0077e..c5de8f1 100755 --- a/script/todo/todo.py +++ b/script/todo/todo.py @@ -1,4 +1,6 @@ #!/usr/bin/env python3 +# © 2021-2025 TechnoLibre (http://www.technolibre.ca) +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl) import datetime import getpass @@ -9,6 +11,14 @@ import shutil import subprocess import sys import time +import zipfile + +new_path = os.path.normpath( + os.path.join(os.path.dirname(__file__), "..", "..") +) +sys.path.append(new_path) + +from script.config import config_file file_error_path = ".erplibre.error.txt" cst_venv_erplibre = ".venv.erplibre" @@ -16,7 +26,7 @@ VERSION_DATA_FILE = os.path.join("conf", "supported_version_erplibre.json") INSTALLED_ODOO_VERSION_FILE = os.path.join( ".repo", "installed_odoo_version.txt" ) -ODOO_VERSION_FILE = os.path.join(".odoo-version") +ODOO_VERSION_FILE = ".odoo-version" ENABLE_CRASH = False CRASH_E = None @@ -27,12 +37,13 @@ try: import click import humanize import openai - from pykeepass import PyKeePass + import todo_file_browser - # TODO implement urwid to improve text user interface # import urwid # TODO implement rich for beautiful print and table # import rich + import todo_upgrade + from pykeepass import PyKeePass except ModuleNotFoundError as e: humanize = None ENABLE_CRASH = True @@ -41,35 +52,48 @@ except ModuleNotFoundError as e: if not ENABLE_CRASH: print("Importation success!") +logging.basicConfig( + format=( + "%(asctime)s,%(msecs)d %(levelname)-8s [%(filename)s:%(lineno)d]" + " %(message)s" + ), + datefmt="%Y-%m-%d:%H:%M:%S", + level=logging.INFO, +) _logger = logging.getLogger(__name__) CONFIG_FILE = "./script/todo/todo.json" -CONFIG_OVERRIDE_FILE = "./private/todo.json" +CONFIG_OVERRIDE_FILE = "./private/todo/todo.json" LOGO_ASCII_FILE = "./script/todo/logo_ascii.txt" class TODO: def __init__(self): + self.dir_path = None self.kdbx = None self.init() + self.file_path = None + self.config_file = config_file.ConfigFile() def init(self): # Get command self.cmd_source_erplibre = "" + self.cmd_source_default = "" exec_path_gnome_terminal = shutil.which("gnome-terminal") if exec_path_gnome_terminal: self.cmd_source_erplibre = ( f"gnome-terminal -- bash -c 'source" f" ./{cst_venv_erplibre}/bin/activate;%s'" ) + self.cmd_source_default = "gnome-terminal -- bash -c '" f"%s'" else: exec_path_tell = shutil.which("osascript") if exec_path_tell: self.cmd_source_erplibre = ( "osascript -e 'tell application \"Terminal\"'" ) - self.cmd_source_erplibre += " -e 'tell application \"System Events\" to keystroke \"PATH\" using {command down}' -e 'delay 0.1' -e 'do script \"" - self.cmd_source_erplibre += f"./{cst_venv_erplibre}/bin/activate; %s\" in front window'" + self.cmd_source_erplibre += " -e 'tell application \"System Events\" to keystroke \"t\" using {command down}' -e 'delay 0.1' -e 'do script \"" + self.cmd_source_erplibre += f"cd {os.getcwd()}; source ./{cst_venv_erplibre}/bin/activate; %s\" in front window'" self.cmd_source_erplibre += " -e 'end tell'" else: self.cmd_source_erplibre = ( @@ -77,7 +101,7 @@ class TODO: ) def run(self): - with open(LOGO_ASCII_FILE) as my_file: + with open(self.config_file.get_logo_ascii_file_path()) as my_file: print(my_file.read()) print("Ouverture de TODO en cours ...") print("🤖 => Entre tes directives par son chiffre et fait Entrée!") @@ -93,12 +117,14 @@ class TODO: status = click.prompt(help_info) except NameError: print("Do") - print("source .venv.erplibre/bin/activate && make") + print(f"source ./{cst_venv_erplibre}/bin/activate && make") sys.exit(1) except ImportError: print("Do") - print("source .venv.erplibre/bin/activate && make") + print(f"source ./{cst_venv_erplibre}/bin/activate && make") sys.exit(1) + except click.exceptions.Abort: + sys.exit(0) print() if status == "0": break @@ -127,7 +153,7 @@ class TODO: if self.kdbx: return self.kdbx # Open file - chemin_fichier_kdbx = self.get_config(["kdbx", "path"]) + chemin_fichier_kdbx = self.config_file.get_config(["kdbx", "path"]) if not chemin_fichier_kdbx: root = tk.Tk() root.withdraw() # Hide the main window @@ -136,10 +162,12 @@ class TODO: filetypes=(("KeepassX files", "*.kdbx"),), ) if not chemin_fichier_kdbx: - _logger.error(f"KDBX is not configured, please fill {CONFIG_FILE}") + _logger.error( + f"KDBX is not configured, please fill {self.config_file.CONFIG_FILE}" + ) return - mot_de_passe_kdbx = self.get_config(["kdbx", "password"]) + mot_de_passe_kdbx = self.config_file.get_config(["kdbx", "password"]) if not mot_de_passe_kdbx: mot_de_passe_kdbx = getpass.getpass( prompt="Entrez votre mot de passe : " @@ -151,25 +179,6 @@ class TODO: self.kdbx = kp return kp - def get_config(self, lst_params): - # Open file - config_file = CONFIG_FILE - if os.path.exists(CONFIG_OVERRIDE_FILE): - config_file = CONFIG_OVERRIDE_FILE - - with open(config_file) as cfg: - dct_data = json.load(cfg) - for param in lst_params: - try: - dct_data = dct_data[param] - except KeyError: - _logger.error( - f"KeyError on file {config_file} with keys" - f" {lst_params}" - ) - return - return dct_data - def execute_prompt_ia(self): while True: help_info = """Commande : @@ -182,7 +191,7 @@ class TODO: kp = self.get_kdbx() if not kp: return - nom_configuration = self.get_config( + nom_configuration = self.config_file.get_config( ["kdbx_config", "openai", "kdbx_key"] ) entry = kp.find_entries_by_title(nom_configuration, first=True) @@ -199,10 +208,12 @@ class TODO: def prompt_execute(self): help_info = """Commande : -[1] RUN Exécuter et installer une instance -[2] EXEC Automatisation - Démonstration des fonctions développées -[3] UPD Mise à jour - Update all developed staging source code +[1] Run - Exécuter et installer une instance +[2] Exec - Automatisation - Démonstration des fonctions développées +[3] Mise à jour - Update all developed staging source code [4] Code - Outil pour développeur +[5] Doc - Recherche de documentation +[6] Database - Outils sur les bases de données [0] Retour """ while True: @@ -226,6 +237,14 @@ class TODO: status = self.prompt_execute_code() if status is not False: return + elif status == "5": + status = self.prompt_execute_doc() + if status is not False: + return + elif status == "6": + status = self.prompt_execute_database() + if status is not False: + return else: print("Commande non trouvée 🤖!") @@ -234,13 +253,13 @@ class TODO: first_installation_input = ( input( - "First system installation? This will process system installation" + "💬 First system installation? This will process system installation" " before (Y/N): " ) .strip() - .upper() + .lower() ) - if first_installation_input == "Y": + if first_installation_input == "y": cmd = "./script/version/update_env_version.py --install" self.executer_commande_live(cmd, source_erplibre=True) print("Wait after OS installation before continue.") @@ -268,124 +287,106 @@ class TODO: ".idea" ): pycharm_configuration_input = ( - input("Open Pycharm? (Y/N): ").strip().upper() + input("💬 Open Pycharm? (Y/N): ").strip().lower() ) - if pycharm_configuration_input == "Y": + if pycharm_configuration_input == "y": pycharm_bin = "pycharm" if has_pycharm else "pycharm-community" - self.executer_commande_live(pycharm_bin, source_erplibre=True) + + cmd = f"cd {os.getcwd()} && {pycharm_bin} ./" + self.executer_commande_live( + cmd, + source_erplibre=False, + single_source_erplibre=False, + new_window=True, + ) print( - "Close Pycharm when processing is done before continue" + "👹 WAIT and Close Pycharm when processing is done before continue" " this guide." ) - # Propose Odoo installation # TODO detect last version supported - odoo_installation_input = ( - input("Install virtual environment? (Y/N): ").strip().upper() + # cmd_intern = "./script/install/install_erplibre.sh" + # TODO maybe update q to only install erplibre from install_locally + # TODO problem installing with q, the script depend on odoo + key_i = 0 + dct_cmd_intern_begin = { + "q": ( + "q", + "q: ERPLibre only with system python without Odoo", + "./script/install/install_erplibre.sh", + ), + "w": ( + "w", + "w: Install all Odoo version with ERPLibre", + "make install_odoo_all_version", + ), + "0": ( + "0", + "0: Quitter", + ), + } + dct_final_cmd_intern = {} + lst_version, lst_version_installed, odoo_installed_version = ( + self.get_odoo_version() ) - if odoo_installation_input == "Y": - # cmd_intern = "./script/install/install_erplibre.sh" - # TODO maybe update q to only install erplibre from install_locally - # TODO problem installing with q, the script depend on odoo - key_i = 0 - dct_cmd_intern_begin = { - "q": ( - "q", - "q: ERPLibre only with system python without Odoo", - "./script/install/install_erplibre.sh", - ), - "w": ( - "w", - "w: Install all Odoo version with ERPLibre", - "make install_odoo_all_version", - ), - } - dct_final_cmd_intern = {} - with open(VERSION_DATA_FILE) as txt: - data_version = json.load(txt) - if not data_version: - raise Exception( - f"Internal error, no Odoo version is supported, please valide file '{VERSION_DATA_FILE}'" - ) + for dct_version in lst_version[::-1]: + key_i += 1 + key_s = str(key_i) + label = f"{key_s}: Odoo {dct_version.get('odoo_version')}" - lst_version_transform = [] - for key, value in data_version.items(): - lst_version_transform.append(value) - value["erplibre_version"] = key - - lst_version_installed = [] - if os.path.exists(INSTALLED_ODOO_VERSION_FILE): - with open(INSTALLED_ODOO_VERSION_FILE) as txt: - lst_version_installed = sorted(txt.read().splitlines()) - - odoo_installed_version = None - if os.path.exists(ODOO_VERSION_FILE): - with open(ODOO_VERSION_FILE) as txt: - odoo_installed_version = f"odoo{txt.read().strip()}" - - # Add odoo version installation on command - lst_version = sorted( - lst_version_transform, key=lambda k: k.get("erplibre_version") + odoo_version = f"odoo{dct_version.get('odoo_version')}" + if odoo_version in lst_version_installed: + label += " - Installed" + if odoo_version == odoo_installed_version: + label += " - Actual" + if dct_version.get("default"): + label += " - Default" + if dct_version.get("is_deprecated"): + label += " - Deprecated" + erplibre_version = dct_version.get("erplibre_version") + dct_cmd_intern_begin[key_s] = ( + key_s, + label, + f"./script/version/update_env_version.py --erplibre_version {erplibre_version} --install_dev", ) - for dct_version in lst_version[::-1]: - key_i += 1 - key_s = str(key_i) - label = f"{key_s}: Odoo {dct_version.get('odoo_version')}" - odoo_version = f"odoo{dct_version.get('odoo_version')}" - if odoo_version in lst_version_installed: - label += " - Installed" - if odoo_version == odoo_installed_version: - label += " - Actual" - if dct_version.get("default"): - label += " - Default" - if dct_version.get("is_deprecated"): - label += " - Deprecated" - erplibre_version = dct_version.get("erplibre_version") - dct_cmd_intern_begin[key_s] = ( - key_s, - label, - f"./script/version/update_env_version.py --erplibre_version {erplibre_version} --install_dev", - ) + # Add final command + dct_cmd_intern = {**dct_cmd_intern_begin, **dct_final_cmd_intern} - # Add final command - dct_cmd_intern = {**dct_cmd_intern_begin, **dct_final_cmd_intern} + # Show command + odoo_version_input = "" + while odoo_version_input not in dct_cmd_intern.keys(): + if odoo_version_input: + print(f"Error, cannot understand value '{odoo_version_input}'") + str_input_dyn_odoo_version = ( + "💬 Choose a version:\n\t" + + "\n\t".join([a[1] for a in dct_cmd_intern.values()]) + + "\nSelect : " + ) + odoo_version_input = ( + input(str_input_dyn_odoo_version).strip().lower() + ) - # Show command - odoo_version_input = "" - while odoo_version_input not in dct_cmd_intern.keys(): - if odoo_version_input: - print( - f"Error, cannot understand value '{odoo_version_input}'" - ) - str_input_dyn_odoo_version = ( - "Choose a version:\n\t" - + "\n\t".join([a[1] for a in dct_cmd_intern.values()]) - + "\nSelect : " - ) - odoo_version_input = ( - input(str_input_dyn_odoo_version).strip().lower() - ) + if odoo_version_input == "0": + return - cmd_intern = dct_cmd_intern.get(odoo_version_input)[2] - print(f"Will execute :\n{cmd_intern}") + cmd_intern = dct_cmd_intern.get(odoo_version_input)[2] + print(f"Will execute :\n{cmd_intern}") - # TODO use external script to detect terminal to use on system - # TODO check script open_terminal_code_generator.sh - # cmd_extern = f"gnome-terminal -- bash -c '{cmd_intern};bash'" - try: - subprocess.run( - cmd_intern, shell=True, executable="/bin/bash", check=True - ) - except subprocess.CalledProcessError as e: - print( - f"Le script Bash «{cmd_intern}» a échoué avec le code de retour {e.returncode}." - ) - print("Wait after installation and open projects by terminal.") - print("make open_terminal") - self.restart_script(str(e)) - else: - print("Nothing to do, you need a fresh installation to continue.") + # TODO use external script to detect terminal to use on system + # TODO check script open_terminal_code_generator.sh + # cmd_extern = f"gnome-terminal -- bash -c '{cmd_intern};bash'" + try: + subprocess.run( + cmd_intern, shell=True, executable="/bin/bash", check=True + ) + except subprocess.CalledProcessError as e: + print( + f"Le script Bash «{cmd_intern}» a échoué avec le code de retour {e.returncode}." + ) + print("Wait after installation and open projects by terminal.") + print("make open_terminal") + self.restart_script(str(e)) def execute_from_configuration( self, dct_instance, exec_run_db=False, ignore_makefile=False @@ -407,7 +408,11 @@ class TODO: makefile_cmd = dct_instance.get("makefile_cmd") if makefile_cmd and not ignore_makefile: - self.executer_commande_live(f"make {makefile_cmd}") + self.executer_commande_live( + f"make {makefile_cmd}", + source_erplibre=False, + single_source_erplibre=True, + ) if exec_run_db: db_name = dct_instance.get("database") @@ -435,7 +440,7 @@ class TODO: # TODO proposer le déploiement à distance # TODO proposer l'exécution de docker # TODO proposer la création de docker - lst_instance = self.get_config(["instance"]) + lst_instance = self.config_file.get_config(["instance"]) help_info = self.fill_help_info(lst_instance) while True: @@ -464,7 +469,7 @@ class TODO: print("Commande non trouvée 🤖!") def prompt_execute_fonction(self): - lst_instance = self.get_config(["function"]) + lst_instance = self.config_file.get_config(["function"]) help_info = self.fill_help_info(lst_instance) while True: @@ -493,8 +498,18 @@ class TODO: # TODO proposer les modules manuelles selon la configuration à mettre à jour # TODO proposer la mise à jour de l'IDE # TODO proposer la mise à jour des git-repo + # TODO faire la mise à jour de ERPLibre + # TODO faire l'upgrade d'un odoo vers un autre - lst_instance = self.get_config(["update_from_makefile"]) + lst_instance = self.config_file.get_config(["update_from_makefile"]) + dct_upgrade_odoo_database = { + "prompt_description": "Upgrade Odoo - Migration Database", + } + lst_instance.append(dct_upgrade_odoo_database) + dct_upgrade_poetry = { + "prompt_description": "Upgrade Poetry - Dependency of Odoo", + } + lst_instance.append(dct_upgrade_poetry) help_info = self.fill_help_info(lst_instance) while True: @@ -502,10 +517,15 @@ class TODO: print() if status == "0": return False + elif status == str(len(lst_instance) - 1): + upgrade = todo_upgrade.TodoUpgrade(self) + upgrade.execute_odoo_upgrade() + elif status == str(len(lst_instance)): + self.upgrade_poetry() else: cmd_no_found = True try: - int_cmd = int(status) + int_cmd = int(status) - 1 if 0 < int_cmd <= len(lst_instance): cmd_no_found = False dct_instance = lst_instance[int_cmd - 1] @@ -514,7 +534,6 @@ class TODO: pass if cmd_no_found: print("Commande non trouvée 🤖!") - return False def prompt_execute_code(self): print("🤖 Qu'avez-vous de besoin pour développer?") @@ -529,7 +548,12 @@ class TODO: # [1] Status Git local et distant # [0] Retour # """ - lst_instance = self.get_config(["code_from_makefile"]) + + lst_instance = self.config_file.get_config(["code_from_makefile"]) + dct_upgrade_odoo_database = { + "prompt_description": "Upgrade Module", + } + lst_instance.append(dct_upgrade_odoo_database) help_info = self.fill_help_info(lst_instance) while True: @@ -537,6 +561,8 @@ class TODO: print() if status == "0": return False + elif status == str(len(lst_instance)): + self.upgrade_module() else: cmd_no_found = True try: @@ -549,7 +575,107 @@ class TODO: pass if cmd_no_found: print("Commande non trouvée 🤖!") - return False + + def prompt_execute_doc(self): + print("🤖 Vous cherchez de la documentation?") + lst_instance = [ + {"prompt_description": "Migration module coverage"}, + {"prompt_description": "What change between version"}, + {"prompt_description": "OCA guidelines"}, + {"prompt_description": "OCA migration Odoo 19 milestone"}, + ] + help_info = self.fill_help_info(lst_instance) + + while True: + status = click.prompt(help_info) + print() + if status == "0": + return False + elif status == "1": + str_version = input( + "Select version to upgrade Odoo CE (5-17) : " + ) + try: + int_version = int(str_version) + print( + "https://oca.github.io/OpenUpgrade/coverage_analysis/modules" + f"{int_version * 10}-{(int_version + 1) * 10}.html" + ) + except ValueError: + print( + "https://oca.github.io/OpenUpgrade/030_coverage_analysis.html" + ) + elif status == "2": + str_version = input( + "Select version to show what change for Odoo CE version 8-18) : " + ) + try: + int_version = int(str_version) + print( + f"https://github.com/OCA/maintainer-tools/wiki/Migration-to-version-{int_version}.0" + ) + except ValueError: + print("https://github.com/OCA/maintainer-tools/wiki") + elif status == "3": + print( + "https://github.com/OCA/odoo-community.org/blob/master/website/Contribution/CONTRIBUTING.rst" + ) + elif status == "4": + print("https://github.com/OCA/maintainer-tools/issues/658") + else: + print("Commande non trouvée 🤖!") + + def prompt_execute_database(self): + print("🤖 Faites des modifications sur les bases de données!") + lst_instance = [ + { + "prompt_description": "Download database to create backup (.zip)" + }, + {"prompt_description": "Restore from backup (.zip)"}, + ] + help_info = self.fill_help_info(lst_instance) + + while True: + status = click.prompt(help_info) + print() + if status == "0": + return False + elif status == "1": + self.download_database_backup_cli() + elif status == "2": + self.restore_from_database() + else: + print("Commande non trouvée 🤖!") + + def get_odoo_version(self): + with open(VERSION_DATA_FILE) as txt: + data_version = json.load(txt) + + if not data_version: + raise Exception( + f"Internal error, no Odoo version is supported, please valide file '{VERSION_DATA_FILE}'" + ) + lst_version_transform = [] + for key, value in data_version.items(): + lst_version_transform.append(value) + value["erplibre_version"] = key + + lst_version_installed = [] + if os.path.exists(INSTALLED_ODOO_VERSION_FILE): + with open(INSTALLED_ODOO_VERSION_FILE) as txt: + lst_version_installed = sorted(txt.read().splitlines()) + + odoo_installed_version = None + if os.path.exists(ODOO_VERSION_FILE): + with open(ODOO_VERSION_FILE) as txt: + odoo_installed_version = f"odoo{txt.read().strip()}" + + # Add odoo version installation on command + lst_version = sorted( + lst_version_transform, key=lambda k: k.get("erplibre_version") + ) + + return lst_version, lst_version_installed, odoo_installed_version def kdbx_get_extra_command_user(self, kdbx_key): lst_value = [] @@ -582,11 +708,17 @@ class TODO: return lst_value def prompt_execute_selenium_and_run_db(self, bd, extra_cmd_web_login=""): - cmd = ( - f'parallel ::: "./run.sh -d {bd}" "sleep' - f' 3;./script/selenium/web_login.py{extra_cmd_web_login}"' + # cmd = ( + # f'parallel ::: "./run.sh -d {bd}" "sleep' + # f' 3;./script/selenium/web_login.py{extra_cmd_web_login}"' + # ) + # self.executer_commande_live(cmd) + cmd_server = f"./run.sh -d {bd};bash" + self.executer_commande_live(cmd_server) + cmd_client = ( + f"sleep 3;./script/selenium/web_login.py{extra_cmd_web_login};bash" ) - self.executer_commande_live(cmd) + self.executer_commande_live(cmd_client) def prompt_execute_selenium(self, command=None, extra_cmd_web_login=""): lst_cmd = [] @@ -609,7 +741,20 @@ class TODO: new_cmd += f' "sleep {1 * i};{cmd}"' self.executer_commande_live(new_cmd) - def executer_commande_live(self, commande, source_erplibre=True): + def executer_commande_live( + self, + commande, + source_erplibre=True, + quiet=False, + single_source_erplibre=False, + new_window=False, + single_source_odoo=False, + source_odoo="", + new_env=None, + return_status_and_command=False, + return_status_and_output=False, + return_status_and_output_and_command=False, + ): """ Exécute une commande et affiche la sortie en direct. @@ -617,6 +762,12 @@ class TODO: commande (str): La commande à exécuter (sous forme de chaîne de caractères). """ + my_env = os.environ.copy() + if new_env: + my_env.update(new_env) + + process_start_time = time.time() + return_status = None if source_erplibre: # commande = f"source ./{cst_venv_erplibre}/bin/activate && " + commande # cmd = ( @@ -624,8 +775,25 @@ class TODO: # f" ./{cst_venv_erplibre}/bin/activate;{commande}'" # ) commande = self.cmd_source_erplibre % commande - print(f"Execute : {commande}") # os.system(f"./script/terminal/open_terminal.sh {commande}") + elif single_source_erplibre: + commande = ( + f"source ./{cst_venv_erplibre}/bin/activate && %s" % commande + ) + elif single_source_odoo: + if not source_odoo and os.path.exists("./.erplibre-version"): + with open("./.erplibre-version") as f: + source_odoo = f.read() + commande = ( + f"source ./.venv.{source_odoo}/bin/activate && {commande}" + ) + elif new_window: + commande = self.cmd_source_default % commande + + print("🏠 ⬇ Execute command :") + print(commande) + lst_output = [] + try: process = subprocess.Popen( commande, @@ -636,6 +804,7 @@ class TODO: text=True, bufsize=1, # Désactive la mise en tampon pour la sortie en direct universal_newlines=True, # Pour traiter les sauts de lignes correctement + env=my_env, ) while True: @@ -643,10 +812,15 @@ class TODO: if not ligne: break print(ligne, end="") + if ( + return_status_and_output + or return_status_and_output_and_command + ): + lst_output.append(ligne) process.wait() # Attendre la fin du process - - if process.returncode != 0: + return_status = process.returncode + if process.returncode != 0 and not quiet: print( "La commande a retourné un code d'erreur :" f" {process.returncode}" @@ -664,6 +838,23 @@ class TODO: ) except Exception as e: print(f"Une erreur s'est produite : {e}") + process_end_time = time.time() + duration_sec = process_end_time - process_start_time + if humanize: + duration_delta = datetime.timedelta(seconds=duration_sec) + humain_time = humanize.precisedelta(duration_delta) + print(f"🏠 ⬆ Executed ({humain_time}) :") + else: + print(f"🏠 ⬆ Executed ({duration_sec:.2f} sec.) :") + print(commande) + print() + if return_status_and_output_and_command: + return return_status, commande, lst_output + if return_status_and_command: + return return_status, commande + if return_status_and_output: + return return_status, lst_output + return return_status def crash_diagnostic(self, e): # TODO show message at start if os.path.exists(file_error_path) @@ -716,6 +907,7 @@ class TODO: import click import humanize import openai + import urwid from pykeepass import PyKeePass except ImportError: print("Rerun and exit") @@ -725,6 +917,163 @@ class TODO: else: self.prompt_install() + def upgrade_module(self): + upgrade = todo_upgrade.TodoUpgrade(self) + upgrade.execute_module_upgrade() + + def upgrade_poetry(self): + # Only show the version to the user + status = self.executer_commande_live( + f"make version", + source_erplibre=False, + ) + # TODO maybe autodetect to update it + git_repo_update_input = input( + "💬 Would you like to fetch all your git repositories, you need it (y/Y) : " + ) + if git_repo_update_input.strip().lower() == "y": + status = self.executer_commande_live( + f"./script/manifest/update_manifest_local_dev.sh", + source_erplibre=False, + ) + + poetry_lock = "./poetry.lock" + try: + os.remove(poetry_lock) + except Exception as e: + pass + odoo_long_version = "" + if os.path.exists("./.erplibre-version"): + with open("./.erplibre-version") as f: + odoo_long_version = f.read() + path_file_odoo_lock = f"./requirement/poetry.{odoo_long_version}.lock" + if odoo_long_version: + try: + os.remove(path_file_odoo_lock) + except Exception as e: + pass + + status = self.executer_commande_live( + f"pip install -r requirement/erplibre_require-ments-poetry.txt && " + f"./script/poetry/poetry_update.py -f", + source_erplibre=False, + single_source_erplibre=False, + single_source_odoo=True, + source_odoo=odoo_long_version, + ) + + if os.path.exists(poetry_lock): + shutil.copy2(poetry_lock, path_file_odoo_lock) + + def restore_from_database(self, show_remote_list=True): + path_image_db = os.path.join(os.getcwd(), "image_db") + print("[1] By filename from image_db") + print(f"[] Browser image_db {path_image_db}") + status = input("💬 Select : ") + if status == "1": + file_name = status + else: + self.dir_path = "" + + file_browser = todo_file_browser.FileBrowser( + path_image_db, self.on_dir_selected + ) + file_browser.run_main_frame() + file_name = os.path.basename(self.dir_path) + print(file_name) + + database_name = input("💬 Database name : ") + if not database_name: + _logger.error("Missing database name") + return + status, lst_output = self.executer_commande_live( + f"python3 ./script/database/db_restore.py -d {database_name} --ignore_cache --image {file_name}", + return_status_and_output=True, + single_source_erplibre=True, + source_erplibre=False, + ) + status = ( + input("💬 Would you like to neutralize database (y/Y)? ") + .strip() + .lower() + ) + if status == "y": + status, lst_output = self.executer_commande_live( + f"./script/addons/update_prod_to_dev.sh {database_name}", + return_status_and_output=True, + single_source_erplibre=True, + source_erplibre=False, + ) + status = ( + input("💬 Would you like to update all addons (y/Y)? ") + .strip() + .lower() + ) + if status == "y": + status, lst_output = self.executer_commande_live( + f"./script/addons/update_addons_all.sh {database_name}", + return_status_and_output=True, + single_source_erplibre=True, + source_erplibre=False, + ) + + def download_database_backup_cli(self, show_remote_list=True): + database_domain = input("Domain Odoo (ex. https://mondomain.com) : ") + if show_remote_list: + status, lst_output = self.executer_commande_live( + f"python3 ./script/database/list_remote.py --raw --odoo-url {database_domain}", + return_status_and_output=True, + single_source_erplibre=True, + source_erplibre=False, + ) + if len(lst_output) > 1: + for index, output in enumerate(lst_output): + print(f"{index + 1} - {output}") + database_name = input("Select id of database :").strip() + elif len(lst_output) == 1: + database_name = lst_output[0].strip() + else: + database_name = input( + "Cannot read remote database, Database name :\n" + ) + else: + database_name = input("Database name :\n") + + timestamp = datetime.datetime.now().strftime("%Y-%m-%d_%Hh%Mm%Ss") + default_output_path = f"./image_db/{database_name}_{timestamp}.zip" + output_path = input( + f"Output path (default: {default_output_path}) : " + ).strip() + if not output_path: + output_path = default_output_path + + master_password = getpass.getpass(prompt="Master password : ") + + cmd = "script/database/download_remote.sh --quiet" + my_env = os.environ.copy() + my_env["MASTER_PWD"] = master_password + my_env["DATABASE_NAME"] = database_name + my_env["OUTPUT_FILE_PATH"] = output_path + my_env["ODOO_URL"] = database_domain + status, cmd_executed = self.executer_commande_live( + cmd, + source_erplibre=False, + return_status_and_command=True, + new_env=my_env, + ) + try: + with zipfile.ZipFile(default_output_path, "r") as zip_ref: + manifest_file_1 = zip_ref.open("manifest.json") + _logger.info( + f"Log file '{default_output_path}' is complete and validated." + ) + except Exception as e: + _logger.error(e) + _logger.error( + f"Failed to read manifest.json from backup file '{default_output_path}'." + ) + return status, output_path, database_name + def restart_script(self, last_error): print("Reboot TODO 🤖...") # os.execv(sys.executable, ['python'] + sys.argv) @@ -754,6 +1103,10 @@ class TODO: print("Error detect at first execution.") print(e) + def on_dir_selected(self, dir_path): + self.dir_path = dir_path + todo_file_browser.exit_program() + if __name__ == "__main__": start_time = time.time() diff --git a/script/todo/todo_file_browser.py b/script/todo/todo_file_browser.py new file mode 100644 index 0000000..c29a614 --- /dev/null +++ b/script/todo/todo_file_browser.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python3 +# © 2021-2025 TechnoLibre (http://www.technolibre.ca) +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl) + +import os + +import urwid + + +class FileBrowser(urwid.WidgetWrap): + def __init__(self, initial_path, callback, open_dir=False): + self.callback = callback + self.current_path = os.path.abspath(initial_path) + self.list_walker = urwid.SimpleFocusListWalker([]) + self.listbox = urwid.ListBox(self.list_walker) + super().__init__(self.listbox) + self.open_dir = open_dir + self.refresh_list() + + def refresh_list(self): + """Updates the list of files and directories.""" + self.list_walker.clear() + self.list_walker.append( + urwid.Button("..", on_press=self.go_up_directory) + ) + if self.open_dir: + self.list_walker.append( + urwid.Button(".", on_press=self.select_directory) + ) + + try: + entries = os.listdir(self.current_path) + entries.sort(key=lambda r: r.lower()) + for entry in entries: + full_path = os.path.join(self.current_path, entry) + if os.path.isdir(full_path): + self.list_walker.append( + urwid.Button(f"{entry}/", on_press=self.open_directory) + ) + elif not self.open_dir: + self.list_walker.append( + urwid.Button(entry, on_press=self.select_file) + ) + except OSError as e: + # Handle directory access errors + self.list_walker.append(urwid.Text(f"Access Error: {e}")) + + def go_up_directory(self, button): + """Moves up one level in the directory hierarchy.""" + parent_path = os.path.dirname(self.current_path) + if parent_path != self.current_path: + self.current_path = parent_path + self.refresh_list() + + def open_directory(self, button): + """Moves into a subdirectory.""" + dirname = button.label[:-1] + new_path = os.path.join(self.current_path, dirname) + if os.path.isdir(new_path): + self.current_path = new_path + self.refresh_list() + + def select_directory(self, button): + """Selects a file and calls the callback function.""" + self.callback(self.current_path) + + def select_file(self, button): + """Selects a file and calls the callback function.""" + filename = button.label + selected_file_path = os.path.join(self.current_path, filename) + self.callback(selected_file_path) + + def run_main_frame(self): + main_frame = urwid.Frame( + body=self, + header=urwid.Text(("header", f"Navigate: {self.current_path}")), + footer=urwid.Text( + ("footer", "Use arrow keys to navigate and Enter to select.") + ), + ) + + palette = [ + ("header", "dark cyan", "black"), + ("footer", "dark cyan", "black"), + ("body", "white", "black"), + ("button", "black", "dark cyan", "standout"), + ("focus", "white", "dark green", "bold"), + ("bold", "bold", "black"), + ] + + loop = urwid.MainLoop(main_frame, palette) + loop.run() + + +def exit_program(): + """Exits the program.""" + raise urwid.ExitMainLoop() diff --git a/script/todo/todo_upgrade.py b/script/todo/todo_upgrade.py new file mode 100755 index 0000000..e58f7b7 --- /dev/null +++ b/script/todo/todo_upgrade.py @@ -0,0 +1,2060 @@ +#!/usr/bin/env python3 +# © 2021-2025 TechnoLibre (http://www.technolibre.ca) +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl) + +import datetime +import json +import logging +import os +import shutil +import sys +import zipfile +from uuid import uuid4 + +import click +import todo_file_browser + +new_path = os.path.normpath( + os.path.join(os.path.dirname(__file__), "..", "..") +) +sys.path.append(new_path) + +from script.git.git_tool import GitTool + +_logger = logging.getLogger(__name__) + +PYTHON_BIN = ".venv.erplibre/bin/python3" +UPGRADE_DATABASE_CONFIG_LOG = ".venv.erplibre/odoo_database_migration_log.json" +# UPGRADE_MODULE_CONFIG_LOG = ".venv.erplibre/odoo_module_migration_log.json" +VENV_NAME_MODULE_MIGRATOR = ".venv" +LST_PATH_OCA_ODOO_MODULE_MIGRATOR = ["script", "OCA_odoo-module-migrator"] +PATH_OCA_ODOO_MODULE_MIGRATOR = "./" + "/".join( + LST_PATH_OCA_ODOO_MODULE_MIGRATOR +) +PATH_VENV_MODULE_MIGRATOR = os.path.join( + PATH_OCA_ODOO_MODULE_MIGRATOR, VENV_NAME_MODULE_MIGRATOR +) +PATH_SOURCE_VENV_MODULE_MIGRATOR = os.path.join( + PATH_VENV_MODULE_MIGRATOR, "bin", "activate" +) +FILENAME_ODOO_VERSION = ".odoo-version" +LOCAL_MANIFEST = os.path.join( + ".repo", "local_manifests", "erplibre_manifest.xml" +) + + +class TodoUpgrade: + def __init__(self, todo): + self.file_path = None + self.dir_path = None + self.todo = todo + self.dct_progression = {} + self.lst_command_executed = [] + self.dct_module_per_version = {} + self.dct_module_per_dct_version_path = {} + + def write_config(self): + if "date_create" not in self.dct_progression.keys(): + self.dct_progression["date_create"] = str(datetime.datetime.now()) + self.dct_progression["date_update"] = str(datetime.datetime.now()) + # Always put command_executed at the end + if "command_executed" in self.dct_progression.keys(): + value = self.dct_progression["command_executed"] + del self.dct_progression["command_executed"] + self.dct_progression["command_executed"] = value + with open(UPGRADE_DATABASE_CONFIG_LOG, "w") as f: + json.dump(self.dct_progression, f, indent=4) + + def on_file_selected(self, file_path): + self.file_path = file_path + todo_file_browser.exit_program() + + def on_dir_selected(self, dir_path): + self.dir_path = dir_path + todo_file_browser.exit_program() + + def execute_module_upgrade(self): + print("Welcome to Odoo module upgrade processus with ERPLibre 🤖") + + with open(UPGRADE_DATABASE_CONFIG_LOG, "r") as f: + try: + old_dct_progression = json.load(f) + self.dct_progression = old_dct_progression + self.lst_command_executed = old_dct_progression.get( + "command_executed" + ) + except json.decoder.JSONDecodeError: + print( + f'⚠️ The config file "{UPGRADE_DATABASE_CONFIG_LOG}" is invalid, ignore it.' + ) + + print( + "Migrate a directory repo to migrate all module, or select a directory module." + ) + print("[m] Migrate one module") + print("[r] Migrate one repo") + print("[] Directory browser") + + lst_dir_path = [] + from_version = 0 + to_version = 0 + + status = input("💬 What do you choose : ").strip().lower() + if status == "m": + print("[p] Path (default)") + print("[n] Name") + + status = input("💬 What do you choose : ").strip().lower() + if status == "n": + module_name = input("💬 Module name : ").strip().lower() + from_version = int( + input("💬 From odoo version (12 to 18) : ").strip() + ) + to_version = int( + input("💬 To odoo version (12 to 18) : ").strip() + ) + self.switch_odoo(from_version) + # TODO detect path + ( + lst_module_missing, + lst_module_duplicate, + lst_module_exist, + lst_module_error, + ) = self.check_addons_exist([module_name], get_all_info=True) + if ( + lst_module_missing + or lst_module_duplicate + or lst_module_error + ): + if lst_module_missing: + print(f"Missing list : {lst_module_missing}") + if lst_module_duplicate: + print(f"Duplicate list : {lst_module_duplicate}") + if lst_module_error: + print(f"Error list : {lst_module_error}") + return + lst_dir_path.extend(lst_module_exist) + else: + self.dir_path = input("💬 Path : ").strip() + elif status == "r": + self.dir_path = input("💬 Path : ").strip() + else: + self.dir_path = None + + if not self.dir_path and not lst_dir_path: + initial_dir = os.getcwd() + + file_browser = todo_file_browser.FileBrowser( + initial_dir, self.on_dir_selected, open_dir=True + ) + file_browser.run_main_frame() + + if not lst_dir_path and not os.path.exists(self.dir_path): + _logger.error(f"Path '{self.dir_path}' not exists.") + return + + if not from_version: + from_version = int( + input("💬 From odoo version (12 to 18) : ").strip() + ) + if not to_version: + to_version = int(input("💬 To odoo version (12 to 18) : ").strip()) + + # TODO ask option direct migration + + lst_version_to_update = [ + (a, a + 1) for a in range(from_version, to_version) + ] + # lst_version_to_update = [(from_version, to_version)] + # for actual_version in range(from_version, to_version): + # next_version = actual_version + 1 + for actual_version, next_version in lst_version_to_update: + set_path_migrate_addons = set() + + if not lst_dir_path: + if os.path.exists( + os.path.join(self.dir_path, "__manifest__.py") + ): + lst_dir_path = [ + [os.path.basename(self.dir_path), self.dir_path] + ] + else: + lst_dir_path = [ + [a, os.path.join(self.dir_path, a)] + for a in os.listdir(self.dir_path) + if os.path.exists( + os.path.join(self.dir_path, a, "__manifest__.py") + ) + ] + + lst_path_git_clone_migrate = [] + lst_module_to_migrate_all = [] + for dir_path in lst_dir_path: + source_manifest_path = os.path.join( + dir_path[1], "__manifest__.py" + ).replace(f"odoo{from_version}.0", f"odoo{actual_version}.0") + + is_dir_module = os.path.exists(source_manifest_path) + if not is_dir_module: + continue + source_module_path = dir_path[1].replace( + f"odoo{from_version}.0", f"odoo{actual_version}.0" + ) + source_addons_path = os.path.dirname(dir_path[1]).replace( + f"odoo{from_version}.0", f"odoo{actual_version}.0" + ) + target_module_path = source_module_path.replace( + f"odoo{actual_version}.0", f"odoo{next_version}.0" + ) + target_manifest_path = source_manifest_path.replace( + f"odoo{actual_version}.0", f"odoo{next_version}.0" + ) + target_addons_path = source_addons_path.replace( + f"odoo{actual_version}.0", f"odoo{next_version}.0" + ) + dct_module = { + "source_module_path": source_module_path, + "source_manifest_path": source_manifest_path, + "source_addons_path": source_addons_path, + "target_module_path": target_module_path, + "target_manifest_path": target_manifest_path, + "target_addons_path": target_addons_path, + "module_name": dir_path[0], + "source_version_odoo": actual_version, + "target_version_odoo": next_version, + } + lst_module_to_migrate_all.append(dct_module) + set_path_migrate_addons.add(target_addons_path) + # lst_path_git_clone_migrate.append(target_addons_path) + # TODO auto detect version from + # TODO if detect from directory, check from repo list + # TODO detect from manifest version + + self.switch_odoo(next_version) + self.install_OCA_odoo_module_migrator() + + self.internal_module_upgrade( + next_version, + lst_module_to_migrate_all, + lst_path_git_clone_migrate, + ) + + for commit_path in set_path_migrate_addons: + cmd = f"./script/code/git_commit_migration_addons_path.py --path {commit_path} --odoo_version {next_version}.0" + self.todo_upgrade_execute(cmd) + print(set_path_migrate_addons) + status = input( + f"💬 Please validate git commit on repos, press to continue : " + ).strip() + + def internal_module_upgrade( + self, + next_version, + lst_module_to_migrate_all, + lst_path_git_clone_migrate, + ): + has_cmd = False + cmd_parallel = "parallel :::" + for dct_module in lst_module_to_migrate_all: + target_addons_path = dct_module.get("target_addons_path") + source_addons_path = dct_module.get("source_addons_path") + module_name = dct_module.get("module_name") + source_version_odoo = f'{dct_module.get("source_version_odoo")}.0' + target_version_odoo = f'{dct_module.get("target_version_odoo")}.0' + source_module_path_to_copy = dct_module.get("source_module_path") + # Prepare git environment for target + if target_addons_path not in lst_path_git_clone_migrate: + lst_path_git_clone_migrate.append(target_addons_path) + self.check_and_clone_source_to_target_migration_code( + next_version, + source_addons_path, + target_addons_path, + ) + + cmd_migration = ( + f"echo 'odoo_module_migrate {module_name}' && " + f"cp -r {source_module_path_to_copy} {target_addons_path} && " + f"cd {PATH_OCA_ODOO_MODULE_MIGRATOR} && " + f"source {VENV_NAME_MODULE_MIGRATOR}/bin/activate && " + f"python -m odoo_module_migrate --directory {target_addons_path} --modules {module_name} " + f"--init-version-name {source_version_odoo} --target-version-name {target_version_odoo} " + f"--no-commit && " + f"cd ~- " + # f"cp -r {source_module_path_to_copy} {target_module_path_to_copy} && " + # f"cd {target_module_path_to_copy} && git commit -am '[MIG] {module_name}: Migration to {target_version_odoo}' && cd ~-" + ) + cmd_parallel += f' "{cmd_migration}"' + has_cmd = True + + if lst_module_to_migrate_all: + if has_cmd: + self.todo_upgrade_execute(cmd_parallel) + print("List of path with migrate code :") + print(lst_path_git_clone_migrate) + print("ℹ To show repo status :\nmake repo_show_status") + input("💬 Check migration code, press to continue : ") + + # source_module_path = dct_module_result.get( + # "source_module_path" + # ) + # if not source_module_path: + # _logger.error( + # f"Missing source module path '{source_module_path}'" + # ) + # else: + # if os.path.exists( + # os.path.join(source_module_path, ".git") + # ): + # self.todo_upgrade_execute( + # f"cd '{source_module_path}' && git stash && cd ~-", + # ) + # + # target_module_path = dct_module_result.get( + # "target_module_path" + # ) + # if not target_module_path: + # _logger.error( + # f"Missing target module path '{target_module_path}'" + # ) + # else: + # # TODO check if has file to commit + # self.todo_upgrade_execute( + # f"cd '{target_module_path}' && git commit -am '[MIG] {len(lst_module_to_migrate_all)} modules: Migration to {next_version}' && cd ~-", + # ) + + # TODO copie to next odoo version + # do commit and continue + # continue migration to loop + + if next_version in [18]: + # TODO need odoo 18, validate python version without switch + status = input( + f"💬 Please validate repo is ready to run upgrade views_migration_18, press to continue : " + ).strip() + # Apply modification with views_migration_18 + has_cmd = False + # cmd_serial = "" + cmd_parallel = "parallel :::" + for path_git_clone_migrate in lst_path_git_clone_migrate: + cmd_migration = ( + f"echo 'views_migration_18 {path_git_clone_migrate}' && " + f"./.venv.odoo18.0_python3.12.10/bin/python ./script/code/odoo_upgrade_code_with_dir_module.py --path {path_git_clone_migrate}" + ) + cmd_parallel += f' "{cmd_migration}"' + # cmd_serial += f"{cmd_migration};" + has_cmd = True + + if has_cmd: + # self.todo_upgrade_execute( + # cmd_serial + # ) + self.todo_upgrade_execute(cmd_parallel) + print("List of module with migration 18 :") + print(lst_module_to_migrate_all) + print("ℹ To show repo status :\nmake repo_show_status") + input("💬 Check migration 18 code, press to continue : ") + + if next_version == 17: + status = input( + f"💬 Please validate repo is ready to run upgrade views_migration_17, press to continue : " + ).strip() + # Apply modification with views_migration_17 + has_cmd = False + cmd_serial = "" + cmd_parallel = "parallel :::" + for dct_module in lst_module_to_migrate_all: + database_migration_17_name = ( + f"migration_odoo_{next_version}_{str(uuid4())[:6]}" + ) + module_name = dct_module.get("module_name") + cmd_migration = ( + f"echo 'views_migration_17 {module_name}' && " + f"./run.sh -d {database_migration_17_name} -i {module_name} --load=base,web,views_migration_17 --dev upgrade --no-http --stop-after-init" + ) + cmd_parallel += f' "{cmd_migration}"' + cmd_serial += f"{cmd_migration};" + has_cmd = True + + if has_cmd: + # self.todo_upgrade_execute( + # cmd_serial + # ) + self.todo_upgrade_execute(cmd_parallel) + print("List of module with migration 17 :") + print(lst_module_to_migrate_all) + print("ℹ To show repo status :\nmake repo_show_status") + input("💬 Check migration 17 code, press to continue : ") + + def execute_odoo_upgrade(self): + # TODO update dev environment for git project + # TODO Redeploy new production after upgrade + # 2 upgrades version = 5 environnement. 0-prod init, 1-dev init, 2-dev01, 3-dev02, 4-prod final + print("Welcome to Odoo database upgrade processus with ERPLibre 🤖") + self.lst_command_executed = [] + self.dct_module_per_version = {} + self.dct_module_per_dct_version_path = {} + default_database_name = "test" + + if os.path.exists(UPGRADE_DATABASE_CONFIG_LOG): + with open(UPGRADE_DATABASE_CONFIG_LOG, "r") as f: + try: + old_dct_progression = json.load(f) + self.dct_progression = { + "migration_file": old_dct_progression.get( + "migration_file" + ), + # More useful to ask this question each time + "target_odoo_version": old_dct_progression.get( + "target_odoo_version" + ), + "date_create": old_dct_progression.get("date_create"), + } + except json.decoder.JSONDecodeError: + print( + f'⚠️ The config file "{UPGRADE_DATABASE_CONFIG_LOG}" is invalid, ignore it.' + ) + + print( + f"✨ Detected migration \"{self.dct_progression.get('migration_file')}\" " + f"to version \"{self.dct_progression.get('target_odoo_version')}\", " + f"please select an option." + ) + + print("[1] Erase progression for a new migration") + print("[2] Reuse database with new process") + print( + "[3] Reuse database without state_4, before looping on next version" + ) + print("[4] Reuse database without configuration") + erase_progression_input = ( + input("💬 Select an option or press to continue : ") + .strip() + .lower() + ) + self.dct_progression = {} + if erase_progression_input in ["2", "3", "4"]: + with open(UPGRADE_DATABASE_CONFIG_LOG, "r") as f: + try: + old_dct_progression = json.load(f) + self.dct_progression = { + "migration_file": old_dct_progression.get( + "migration_file" + ), + # More useful to ask this question each time + # "target_odoo_version": old_dct_progression.get( + # "target_odoo_version" + # ), + "date_create": old_dct_progression.get( + "date_create" + ), + } + for key, value in old_dct_progression.items(): + if erase_progression_input == "3": + if ( + key.startswith("state_0") + or key.startswith("state_1") + or key.startswith("state_2") + or key.startswith("state_3") + ): + self.dct_progression[key] = value + if key.startswith(f"config_state") and not ( + key.startswith(f"config_state_0") + or key.startswith(f"config_state_1") + or key.startswith(f"config_state_2") + or key.startswith(f"config_state_3") + ): + continue + if ( + key.startswith("config_") + and erase_progression_input != "4" + ): + self.dct_progression[key] = value + # Force to search missing module to fill dict + self.dct_progression[ + "state_0_search_missing_module" + ] = False + except json.decoder.JSONDecodeError: + print( + f"⚠️ The config file '{UPGRADE_DATABASE_CONFIG_LOG}' is invalid, ignore it." + ) + + self.write_config() + elif erase_progression_input not in ["1"]: + with open(UPGRADE_DATABASE_CONFIG_LOG, "r") as f: + try: + self.dct_progression = json.load(f) + except json.decoder.JSONDecodeError: + print( + f"⚠️ The config file '{UPGRADE_DATABASE_CONFIG_LOG}' is invalid, ignore it." + ) + + if "migration_file" in self.dct_progression: + self.file_path = self.dct_progression["migration_file"] + else: + print("") + print("Select the zip file of you database backup.") + + self.file_path = input( + "💬 Give the path of file, or empty to use a File Browser, or type 'remote' to download from production : " + ) + if not self.file_path.strip(): + self.file_path = None + if not self.file_path: + initial_dir = os.path.join(os.getcwd(), "image_db") + file_browser = todo_file_browser.FileBrowser( + initial_dir, self.on_file_selected + ) + file_browser.run_main_frame() + elif self.file_path == "remote": + status, self.file_path, default_database_name = ( + self.todo.download_database_backup_cli() + ) + if status: + _logger.error( + "Cannot retrieve database from remote, please retry migration." + ) + return + + self.dct_progression["migration_file"] = self.file_path + self.write_config() + + print(f"✅ Open file {self.file_path}") + with zipfile.ZipFile(self.file_path, "r") as zip_ref: + manifest_file_1 = zip_ref.open("manifest.json") + json_manifest_file_1 = json.load(manifest_file_1) + odoo_actual_version = json_manifest_file_1.get("version") + print(f"✅ Detect version Odoo CE '{odoo_actual_version}'.") + + # print("What is your actual Odoo version?") + lst_version, lst_version_installed, odoo_installed_version = ( + self.todo.get_odoo_version() + ) + + lst_odoo_version = [ + {"prompt_description": a.get("odoo_version")} + for a in lst_version + if float(a.get("odoo_version")) > float(odoo_actual_version) + ] + help_info = self.todo.fill_help_info(lst_odoo_version) + + if "target_odoo_version" in self.dct_progression: + odoo_target_version = self.dct_progression["target_odoo_version"] + else: + print("💬 Which version do you want to upgrade to?") + odoo_target_version = None + cmd_no_found = True + while cmd_no_found: + status = click.prompt(help_info) + try: + int_cmd = int(status) + if 0 < int_cmd <= len(lst_odoo_version): + cmd_no_found = False + odoo_target_version = lst_odoo_version[ + int_cmd - 1 + ].get("prompt_description") + except ValueError: + pass + if cmd_no_found: + print("Commande non trouvée 🤖!") + + self.dct_progression["target_odoo_version"] = odoo_target_version + self.write_config() + + # Search nb diff to use range + start_version = int(float(odoo_actual_version)) + end_version = int(float(odoo_target_version)) + range_version = range(start_version, end_version) + lst_module = sorted( + list(set(json_manifest_file_1.get("modules").keys())) + ) + self.dct_module_per_version[start_version] = lst_module + self.dct_progression["dct_module_per_version"] = ( + self.dct_module_per_version + ) + self.dct_progression["lst_module_per_version_origin"] = lst_module + # TODO need support minor version, example 18.2, the .2 (no need for OCE OCB) + + print("✨ Show documentation version :") + # TODO Generate it locally and show it if asked + + for next_version in range_version: + print( + f"https://oca.github.io/OpenUpgrade/coverage_analysis/modules{next_version*10}-{(next_version+1)*10}.html" + ) + + # ⚠️ ℹ 💬 ❗ 🔷 ✨ 🟦 🔹 🔵 ⟳ ⧖ ⚙ ✔ ✅ ❌ ⏵ ⏸ ⏹ ◆ ◇ … ➤ ⚑ ★ ☆ ☰ ⬍ ⍟ ⊗ ⌘ ⏻ ⍰ + msg = "0 - Inspect zip" + print(f"🔷 {msg}") + self.add_comment_progression(msg) + print("✅ -> Search odoo version") + print("✅ -> Find good environment, read the .zip file") + + is_state_4_reach_open_upgrade = self.dct_progression.get( + "state_4_reach_open_upgrade" + ) + + if not is_state_4_reach_open_upgrade and not self.dct_progression.get( + "state_0_install_odoo" + ): + lst_diff_version = sorted( + list( + set([f"odoo{a}.0" for a in range_version]).difference( + set(lst_version_installed) + ) + ) + ) + for odoo_version_to_install in lst_diff_version: + iter_range_version = odoo_version_to_install.replace( + "odoo", "" + ).replace(".0", "") + want_continue = input( + f"💬 Would you like to install '{odoo_version_to_install}' (y/Y) : " + ) + if want_continue.strip().lower() != "y": + return + self.todo_upgrade_execute( + f"make install_odoo_{iter_range_version}" + ) + + if not os.path.isfile(FILENAME_ODOO_VERSION): + print( + "⚠️ You need an installed system before continue, check your Odoo installation." + ) + return + + self.dct_progression["state_0_install_odoo"] = True + self.write_config() + # not self.dct_progression.get("state_0_switch_odoo") + if not is_state_4_reach_open_upgrade: + self.switch_odoo(odoo_actual_version) + # self.dct_progression["state_0_switch_odoo"] = True + # self.write_config() + + print("✅ -> Install environment if missing") + + if not self.dct_progression.get("state_0_search_missing_module"): + self.switch_odoo(odoo_actual_version) + dct_bd_modules = json_manifest_file_1.get("modules") + lst_module_to_check = [a for a in dct_bd_modules.keys()] + ( + lst_module_missing, + lst_module_duplicate, + lst_module_exist, + lst_module_error, + ) = self.check_addons_exist(lst_module_to_check, get_all_info=True) + if not lst_module_missing: + lst_module_missing = [] + dct_module_exist = {} + if not lst_module_exist: + lst_module_exist = [] + else: + for item_lst_module_exist in lst_module_exist: + dct_module_exist[item_lst_module_exist[0]] = ( + item_lst_module_exist[1].replace(os.getcwd(), ".") + ) + if not lst_module_duplicate: + lst_module_duplicate = [] + + lst_module_missing = sorted(list(set(lst_module_missing))) + self.dct_progression["len_lst_module_missing"] = len( + lst_module_missing + ) + self.dct_progression["lst_module_missing"] = lst_module_missing + self.dct_progression["len_dct_module_exist"] = len( + lst_module_exist + ) + self.dct_progression["dct_module_exist"] = dct_module_exist + self.dct_progression["len_lst_module_duplicate"] = len( + lst_module_duplicate + ) + + self.dct_progression["lst_module_duplicate"] = lst_module_duplicate + self.write_config() + if lst_module_missing or lst_module_duplicate: + print("Cannot setup environment to begin.") + if lst_module_missing: + print("Missing module :") + print(lst_module_missing) + if lst_module_duplicate: + print("Duplicate module :") + print(lst_module_duplicate) + want_continue = input( + "💬 Detect error missing/duplicate module init, do you want to continue? (Y/N): " + ) + if want_continue.strip().lower() != "y": + return + + self.dct_progression["state_0_search_missing_module"] = True + self.write_config() + else: + # TODO fill from config + lst_module_missing = [] + + print("✅ -> Search missing module") + + print( + "❌ -> Install missing module, do a research or ask to uninstall it (can break data)" + ) + + msg = "1 - Import database from zip" + print(f"🔷 {msg}") + self.add_comment_progression(msg) + + database_name = self.dct_progression.get("config_database_name") + if not database_name: + database_name = ( + input( + f"💬 Witch database name do you want to work with? Default ({default_database_name}) : " + ).strip() + or default_database_name + ) + self.dct_progression["config_database_name"] = database_name + self.write_config() + + do_neutralize = False + if not self.dct_progression.get("state_1_neutralize_database"): + print("[1] Ignore neutralize database") + wait_continue = ( + input("💬 Neutralize database, press to continue : ") + .strip() + .lower() + ) + if wait_continue != "1": + do_neutralize = True + database_name += "_neutralize" + self.dct_progression["config_database_name"] = database_name + self.write_config() + + print(f"★ Work with database '{database_name}'") + + if not self.dct_progression.get("state_1_restore_database"): + file_name = os.path.basename(self.file_path) + image_db_file_path = os.path.join("image_db", file_name) + if os.path.exists(image_db_file_path): + if not shutil._samefile(self.file_path, image_db_file_path): + status_overwrite_image_db = input( + f"🤖 will copy '{self.file_path}' to '{image_db_file_path}', " + f"a file already exist, do you want to continue (y/Y) : " + ).strip() + if status_overwrite_image_db.lower() != "y": + return + os.remove(image_db_file_path) + shutil.copy(self.file_path, image_db_file_path) + + status, cmd_executed = self.todo_upgrade_execute( + f"./script/database/db_restore.py --database {database_name} --image {file_name} --ignore_cache", + single_source_odoo=True, + ) + if not status: + self.dct_progression["state_1_restore_database"] = True + self.write_config() + + print("✅ -> Restore database") + + print("✅ -> Neutralize database") + if do_neutralize: + status, cmd_executed = self.todo_upgrade_execute( + f"./script/addons/update_prod_to_dev.sh {database_name}", + single_source_odoo=True, + ) + self.dct_progression["state_1_neutralize_database"] = True + self.write_config() + if not status: + self.dct_progression["state_1_neutralize_database"] = True + self.write_config() + else: + self.dct_progression["state_1_neutralize_database"] = True + self.write_config() + + config_state_1_uninstall_module = self.dct_progression.get( + "config_state_1_uninstall_module" + ) + config_state_1_install_module = self.dct_progression.get( + "config_state_1_install_module" + ) + + if not is_state_4_reach_open_upgrade: + lst_module_to_uninstall = [] + uninstall_module_list_file = os.path.join( + "script", + "odoo", + "migration", + f"uninstall_module_list_odoo{start_version * 10}_to_odoo{(start_version + 1) * 10}.txt", + ) + if os.path.exists(uninstall_module_list_file): + with open(uninstall_module_list_file, "r") as f: + lst_module_to_uninstall = [ + a.strip() for a in f.readline().split() + ] + + if config_state_1_uninstall_module: + lst_module_to_uninstall = ( + lst_module_to_uninstall + config_state_1_uninstall_module + ) + + if lst_module_to_uninstall: + self.uninstall_from_database( + lst_module_to_uninstall, database_name, start_version + ) + self.dct_progression["state_1_uninstall_module"] = True + self.write_config() + + self.dct_progression["config_state_1_uninstall_module"] = ( + config_state_1_uninstall_module + ) + + self.write_config() + + print("✅ -> Uninstall module") + + print("✅ -> install module") + if not is_state_4_reach_open_upgrade: + lst_module_to_install = [] + if config_state_1_install_module: + lst_module_to_install = ( + lst_module_to_install + config_state_1_install_module + ) + if lst_module_to_install: + self.install_from_database( + lst_module_to_install, database_name, start_version + ) + self.dct_progression["state_1_install_module"] = True + self.write_config() + self.dct_progression["config_state_1_install_module"] = ( + config_state_1_install_module + ) + self.write_config() + + msg = "2 - Succeed update all addons" + print(f"🔷 {msg}") + self.add_comment_progression(msg) + + if not self.dct_progression.get("state_2_update_all"): + status, cmd_executed = self.todo_upgrade_execute( + f"./script/addons/update_addons_all.sh {database_name}", + single_source_odoo=True, + ) + if not status: + self.dct_progression["state_2_update_all"] = True + self.write_config() + + msg = "3 - Clean up database before data migration" + print(f"🔷 {msg}") + self.add_comment_progression(msg) + + if not self.dct_progression.get("state_3_install_clean_database"): + status, cmd_executed = self.todo_upgrade_execute( + f"./script/addons/install_addons.sh {database_name} database_cleanup", + single_source_odoo=True, + ) + if not status: + self.dct_progression["state_3_install_clean_database"] = True + self.write_config() + + if not self.dct_progression.get("state_3_clean_database"): + print( + "✨ Aller dans «configuration/Technique/Nettoyage.../Purger» les modules obsolètes" + ) + status = input( + "💬 Did you finish to clean database? Press y/Y to open server with selenium, else ignore it : " + ).strip() + + if status.lower().strip() == "y": + self.todo.prompt_execute_selenium_and_run_db(database_name) + status = input("💬 Press to continue state.3 : ").strip() + + self.dct_progression["state_3_clean_database"] = True + self.write_config() + + self.install_OCA_odoo_module_migrator() + + msg = "4 - Upgrade version with OpenUpgrade" + print(f"🔷 {msg}") + self.add_comment_progression(msg) + + self.dct_progression["state_4_reach_open_upgrade"] = True + self.write_config() + lst_next_version = [ + a for a in range(start_version + 1, end_version + 1) + ] + lst_database_name_upgrade = [ + f"{database_name}_upgrade_{str(a)}" for a in lst_next_version + ] + # Setup lst_switch_odoo + lst_clone_odoo = self.dct_progression.get( + "state_4_clone_odoo_lst", [False] * len(lst_next_version) + ) + lst_switch_odoo = self.dct_progression.get( + "state_4_switch_odoo_lst", [False] * len(lst_next_version) + ) + lst_module_migrate_odoo = self.dct_progression.get( + "state_4_module_migrate_odoo_lst", [False] * len(lst_next_version) + ) + lst_module_uninstall_module = self.dct_progression.get( + "state_4_uninstall_module", [False] * len(lst_next_version) + ) + lst_module_install_module = self.dct_progression.get( + "state_4_install_module", [False] * len(lst_next_version) + ) + lst_module_search_missing_module = self.dct_progression.get( + "state_4_search_missing_module", [False] * len(lst_next_version) + ) + + nb_missing_value_switch_odoo = abs( + len(lst_switch_odoo) - len(lst_next_version) + ) + if nb_missing_value_switch_odoo: + lst_switch_odoo += [False] * nb_missing_value_switch_odoo + + # Setup lst_upgrade_odoo + lst_upgrade_odoo = self.dct_progression.get( + "state_4_upgrade_odoo_lst", [[]] * len(lst_next_version) + ) + lst_fix_migration_odoo = self.dct_progression.get( + "state_4_fix_migration_odoo_lst", [[]] * len(lst_next_version) + ) + nb_missing_value_upgrade_odoo = abs( + len(lst_upgrade_odoo) - len(lst_next_version) + ) + if nb_missing_value_upgrade_odoo: + lst_upgrade_odoo += [[]] * nb_missing_value_upgrade_odoo + + database_name_upgrade = None + lst_module_missing_next_version = [] + lst_module_to_delete = [] + lst_module_to_delete_last_version = [] + for index, next_version in enumerate(lst_next_version): + # Reinit the list + lst_module_missing_last_version = lst_module_missing_next_version[ + : + ] + lst_module_to_delete_last_version.extend(lst_module_to_delete) + lst_module_to_delete = [] + + msg = f"4.{index} - Ready to work with version {next_version}" + self.add_comment_progression(msg) + + option_comment = 0 + msg = f"4.{index}.{chr(option_comment + 65)} - Search updated module list to next version" + self.add_comment_progression(msg) + + if not database_name_upgrade: + last_database_name = database_name + else: + last_database_name = database_name_upgrade + database_name_upgrade = lst_database_name_upgrade[index] + lst_module_to_uninstall = [] + lst_module_to_install = [] + lst_module_to_analyse = self.get_rename_module( + self.dct_module_per_version[next_version - 1], + next_version, + ) + self.dct_module_per_version[next_version] = sorted( + list(set(lst_module_to_analyse)) + ) + self.dct_progression["dct_module_per_version"] = ( + self.dct_module_per_version + ) + + option_comment += 1 + msg = f"4.{index}.{chr(option_comment + 65)} - Clone Odoo" + self.add_comment_progression(msg) + + if not os.path.exists(f"odoo{next_version}.0/addons/addons"): + os.makedirs(f"odoo{next_version}.0/addons/addons") + + if not lst_clone_odoo[index]: + self.switch_odoo(next_version - 1) + + print( + f"⧖ -> Clone to odoo.'{next_version}', from '{database_name}' to '{database_name_upgrade}'." + ) + # Delete if exist database + self.todo_upgrade_execute( + f"./script/database/db_restore.py -d {database_name_upgrade} --only_drop", + ) + + # Duplicate database + cmd_clone_database = f"./odoo_bin.sh db --clone --from_database {last_database_name} --database {database_name_upgrade}" + self.todo_upgrade_execute(cmd_clone_database) + + lst_clone_odoo[index] = True + self.dct_progression["state_4_clone_odoo_lst"] = lst_clone_odoo + self.write_config() + print(f"✅ -> Clone Odoo{next_version} done") + else: + print(f"✅ -> Clone Odoo{next_version} - nothing") + + option_comment += 1 + msg = f"4.{index}.{chr(option_comment + 65)} - Uninstall module" + self.add_comment_progression(msg) + + config_state_4_uninstall_module = self.dct_progression.get( + "config_state_4_uninstall_module", + [False] * len(lst_next_version), + ) + + if not lst_module_uninstall_module[index]: + lst_module_to_uninstall = config_state_4_uninstall_module[ + index + ] + + if lst_module_to_uninstall: + self.uninstall_from_database( + lst_module_to_uninstall, + database_name_upgrade, + next_version - 1, + ) + lst_module_uninstall_module[index] = True + self.dct_progression["state_4_module_migrate_odoo_lst"] = ( + lst_module_uninstall_module + ) + self.write_config() + + self.dct_progression["config_state_4_uninstall_module"] = ( + config_state_4_uninstall_module + ) + self.dct_progression["state_4_uninstall_module"] = ( + lst_module_uninstall_module + ) + self.write_config() + + option_comment += 1 + msg = f"4.{index}.{chr(option_comment + 65)} - Install module" + self.add_comment_progression(msg) + + config_state_4_install_module = self.dct_progression.get( + "config_state_4_install_module", + [False] * len(lst_next_version), + ) + + # Special case to install module to fix migration + if ( + next_version == 13 + and "dms" in lst_module_to_analyse + and "muk_dms" + not in self.dct_progression.get("dct_module_exist", {}).keys() + ): + # Force install dms into odoo 12 + if not config_state_4_install_module[index]: + config_state_4_install_module[index] = ["dms"] + elif "dms" not in config_state_4_install_module[index]: + config_state_4_install_module[index].append("dms") + + if not lst_module_install_module[index]: + lst_module_to_install = config_state_4_install_module[index] + if not lst_module_to_install: + lst_module_to_install = [] + + if lst_module_to_install: + self.install_from_database( + lst_module_to_install, + database_name_upgrade, + next_version - 1, + ) + lst_module_install_module[index] = True + self.dct_progression["state_4_module_migrate_odoo_lst"] = ( + lst_module_install_module + ) + self.write_config() + + self.dct_progression["config_state_4_install_module"] = ( + config_state_4_install_module + ) + self.dct_progression["state_4_install_module"] = ( + lst_module_install_module + ) + self.write_config() + + option_comment += 1 + msg = f"4.{index}.{chr(option_comment + 65)} - Switch Odoo" + self.add_comment_progression(msg) + + if not lst_switch_odoo[index]: + self.switch_odoo(next_version) + lst_switch_odoo[index] = True + self.dct_progression["state_4_switch_odoo_lst"] = ( + lst_switch_odoo + ) + self.write_config() + print(f"✅ -> Switch Odoo{next_version} done with update") + else: + print(f"✅ -> Switch Odoo{next_version} - nothing") + + lst_state_4_module_migrate_code = self.dct_progression.get( + "config_state_4_module_to_migrate_code", + [[]] * len(lst_next_version), + ) + if ( + "config_state_4_module_to_migrate_code" + not in self.dct_progression.keys() + ): + self.dct_progression[ + "config_state_4_module_to_migrate_code" + ] = lst_state_4_module_migrate_code + lst_module_to_migrate = lst_state_4_module_migrate_code[index] + + option_comment += 1 + msg = ( + f"4.{index}.{chr(option_comment + 65)} - Search missing module" + ) + self.add_comment_progression(msg) + + if not lst_module_search_missing_module[index]: + lst_module_to_analyse_updated = [] + for bd_module in lst_module_to_analyse: + if ( + lst_module_to_uninstall + and bd_module in lst_module_to_uninstall + ): + # Ignore check if uninstall before + continue + lst_module_to_analyse_updated.append(bd_module) + + # TODO remove from list past module deleted + lst_module_to_check = [ + a + for a in lst_module_to_analyse_updated + if a not in lst_module_to_delete_last_version + ] + ( + lst_module_missing_next_version, + lst_module_duplicate_next_version, + ) = self.check_addons_exist(lst_module_to_check) + + lst_module_missing_next_version = sorted( + list(set(lst_module_missing_next_version)) + ) + + self.dct_progression["state_4_len_lst_module_missing"] = len( + lst_module_missing_next_version + ) + self.dct_progression["state_4_lst_module_missing"] = ( + lst_module_missing_next_version + ) + + lst_module_duplicate = sorted( + list(set(lst_module_duplicate_next_version)) + ) + + self.dct_progression["state_4_len_lst_module_duplicate"] = len( + lst_module_duplicate + ) + self.dct_progression["state_4_lst_module_duplicate"] = ( + lst_module_duplicate + ) + + if lst_module_duplicate: + print(f"Duplicate module into odoo{next_version} : ") + print(lst_module_duplicate) + input( + f"💬 Detect error duplicate module, manage this problem manually and press to continue." + ) + # if lst_module_missing_next_version and not lst_module_to_migrate: + if lst_module_missing_next_version: + # TODO support when lst_module_to_migrate is fill + lst_module_to_migrate = [] + print( + f"👹 Detect error missing module, missing module into odoo{next_version} :" + ) + for index_missing_module, module_missing in enumerate( + lst_module_missing_next_version + ): + old_path = self.dct_progression.get( + "dct_module_exist", {} + ).get(module_missing) + print( + f"[{index_missing_module}] {module_missing} - {old_path}" + ) + print("[a] All list above") + print("[e] Add extra custom") + + want_continue = ( + input( + f"💬 Enumerate missing module separate by coma to delete it" + f". The others will be migrate : " + ) + .strip() + .lower() + ) + + is_delete_all = False + + if want_continue: + lst_want_continue = [ + a.strip() for a in want_continue.split(",") + ] + if "a" in lst_want_continue: + is_delete_all = True + lst_module_to_delete = [ + lst_module_missing_next_version[a] + for a in range( + len(lst_module_missing_next_version) + ) + ] + else: + # TODO show error if the index is wrong + lst_want_continue_number = [ + int(a) + for a in lst_want_continue + if a.isdigit() + ] + lst_module_to_delete = [ + lst_module_missing_next_version[a] + for a in lst_want_continue_number + if 0 + <= a + < len(lst_module_missing_next_version) + ] + if len(lst_module_to_delete) == len( + lst_module_missing_next_version + ): + is_delete_all = True + + if next_version == 15: + lst_module_to_delete.append("users_default_groups") + lst_module_to_delete.append( + "web_editor_backend_context" + ) + lst_module_to_delete.append( + "website_google_analytics_fixed" + ) + elif next_version == 17: + lst_module_to_delete.append( + "export_delete_login_log" + ) + elif next_version == 18: + lst_module_to_delete.append( + "base_attachment_object_storage" + ) + lst_module_to_delete.append( + "user_password_strength" + ) + + if "e" in lst_want_continue: + want_continue = ( + input( + f"💬 Enumerate module name to delete, separate by coma : " + ) + .strip() + .lower() + ) + lst_to_extend = [ + a.strip() for a in want_continue.split(",") + ] + lst_module_to_delete.extend(lst_to_extend) + + if lst_module_to_delete: + msg = f"4.{index}.{chr(option_comment + 65)}.option - Choose delete missing module" + self.add_comment_progression(msg) + + self.switch_odoo(next_version - 1) + + if lst_module_to_delete: + # Delete if exist database + self.todo_upgrade_execute( + f"./script/database/db_restore.py -d {database_name_upgrade} --only_drop", + ) + # Duplicate database + cmd_clone_database = f"./odoo_bin.sh db --clone --from_database {last_database_name} --database {database_name_upgrade}" + self.todo_upgrade_execute(cmd_clone_database) + self.uninstall_from_database( + lst_module_to_delete, + database_name_upgrade, + next_version, + ) + self.install_from_database( + lst_module_to_install, + database_name_upgrade, + next_version - 1, + ) + + if not is_delete_all: + msg = f"4.{index}.{chr(option_comment + 65)}.option - Choose auto-fix (not implemented yet)" + self.add_comment_progression(msg) + + lst_module_to_migrate_code = set( + lst_module_missing_next_version + ) - set(lst_module_to_delete) + ( + lst_module_missing_last, + lst_module_duplicate_last, + lst_module_exist_last, + lst_module_error_last, + ) = self.check_addons_exist( + lst_module_to_migrate_code, get_all_info=True + ) + + if lst_module_missing_last: + print( + f"Error missing module : {lst_module_missing_last}" + ) + if lst_module_duplicate_last: + print( + f"Error duplicate module : {lst_module_duplicate_last}" + ) + if lst_module_error_last: + print( + f"Error error module : {lst_module_error_last}" + ) + + if lst_module_exist_last: + odoo_name_last_version = ( + f"odoo{next_version - 1}.0" + ) + odoo_name_actual_version = f"odoo{next_version}.0" + for ( + module_name, + module_path, + ) in lst_module_exist_last: + module_dir_path = os.path.dirname(module_path) + module_dir_path_new_version = ( + module_dir_path.replace( + odoo_name_last_version, + odoo_name_actual_version, + ) + ) + module_dir_path_manifest = os.path.join( + module_path, "__manifest__.py" + ) + module_dir_new_version = os.path.join( + module_dir_path_new_version, module_name + ) + module_dir_new_version_manifest = os.path.join( + module_dir_new_version, "__manifest__.py" + ) + dct_module_to_migrate_module = { + "source_module_path": module_path, + "source_manifest_path": module_dir_path_manifest, + "source_addons_path": module_dir_path, + "target_module_path": module_dir_new_version, + "target_manifest_path": module_dir_new_version_manifest, + "target_addons_path": module_dir_path_new_version, + "module_name": module_name, + "source_version_odoo": next_version - 1, + "target_version_odoo": next_version, + } + # TODO move this into config + lst_module_to_migrate.append( + dct_module_to_migrate_module + ) + + self.dct_progression[ + "config_state_4_module_to_migrate_code" + ][index] = lst_module_to_migrate + self.write_config() + + self.switch_odoo(next_version) + # TODO auto-fix + # TODO try to migrate module, find in previous version, application la migration vers une nouvelle version + # TODO ajouté menu todo qui permet de faire une migration d'un module et migrer le générateur de code. + # TODO when check module, reminder provenance + # TODO implement asyncio instead of parallel + # TODO detect when duplicate path module ou module manquant, prendre décision qui ont efface si dupliqué + # TODO pourquoi web_ir_actions_act_multi est doublé dans odoo 13 + + lst_module_search_missing_module[index] = True + self.dct_progression["state_4_search_missing_module"] = ( + lst_module_search_missing_module + ) + self.write_config() + option_comment += 1 + msg = f"4.{index}.{chr(option_comment + 65)} - Migrate module" + self.add_comment_progression(msg) + lst_path_git_clone_migrate = [] + + if not lst_module_migrate_odoo[index]: + # TODO Searching module + # search addons/addons + # search read manifest and detect branch difference, manifest into private? + # Extract module name and run migration to another list + # Maybe check if already exist and show list or continue with overwrite + # Expliquer pourquoi on ne fait pas le oca-port, c'est + + config_migrate_repo = self.dct_progression.get( + "config_migrate_repo", False + ) + self.dct_progression["config_migrate_repo"] = ( + config_migrate_repo + ) + + if config_migrate_repo: + dct_module_result = self.search_module_to_move( + next_version - 1, next_version + ) + + # TODO code migration + # git stash + # call odoo-module-migrate, without commit + + source_module_path = dct_module_result.get( + "source_module_path" + ) + if not source_module_path: + _logger.error( + f"Missing source module path '{source_module_path}'" + ) + else: + if os.path.exists( + os.path.join(source_module_path, ".git") + ): + self.todo_upgrade_execute( + f"cd '{source_module_path}' && git stash && cd ~-" + ) + + target_module_path = dct_module_result.get( + "target_module_path" + ) + if not target_module_path: + _logger.error( + f"Missing target module path '{target_module_path}'" + ) + else: + if os.path.exists( + os.path.join(target_module_path, ".git") + ): + self.todo_upgrade_execute( + f"cd '{target_module_path}' && git stash && cd ~-" + ) + + lst_module_to_migrate_all = dct_module_result.get( + "lst_module", [] + ) + else: + lst_module_to_migrate_all = [] + + # TODO remove duplicate au lieu d'extend + lst_module_to_migrate_all.extend(lst_module_to_migrate) + + self.internal_module_upgrade( + next_version, + lst_module_to_migrate_all, + lst_path_git_clone_migrate, + ) + + lst_module_migrate_odoo[index] = True + self.dct_progression["state_4_module_migrate_odoo_lst"] = ( + lst_module_migrate_odoo + ) + self.write_config() + + print(f"✅ -> Module upgrade Odoo{next_version} done") + else: + print(f"✅ -> Module upgrade Odoo{next_version} - nothing") + + option_comment += 1 + msg = f"4.{index}.{chr(option_comment + 65)} - Fix migrate code" + self.add_comment_progression(msg) + + if not lst_fix_migration_odoo[index]: + print("") + file_path_fix_migration = os.path.join( + "script", + "odoo", + "migration", + f"fix_migration_odoo{(next_version-1)*10}_to_odoo{next_version*10}.py", + ) + if os.path.exists(file_path_fix_migration): + self.todo_upgrade_execute( + f"cat ./{file_path_fix_migration} | ./odoo{next_version}.0/odoo/odoo-bin shell -d {database_name_upgrade}", + single_source_odoo=True, + ) + + lst_fix_migration_odoo[index] = file_path_fix_migration + self.dct_progression["state_4_fix_migration_odoo_lst"] = ( + lst_fix_migration_odoo + ) + self.write_config() + print(f"✅ -> Fix migration Odoo{next_version} done") + else: + print( + f"✅ -> Fix migration Odoo{next_version} - no fix to execute" + ) + else: + print(f"✅ -> Fix migration Odoo{next_version} - nothing") + + for path_git_clone_migrate in lst_path_git_clone_migrate: + cmd = f"./script/code/git_commit_migration_addons_path.py --path {path_git_clone_migrate} --odoo_version {next_version}.0" + self.todo_upgrade_execute(cmd) + + option_comment += 1 + msg = f"4.{index}.{chr(option_comment + 65)} - Migrate database" + self.add_comment_progression(msg) + + if not lst_upgrade_odoo[index]: + path_addons_openupgrade = os.path.join( + os.getcwd(), f"odoo{next_version}.0", "OCA_OpenUpgrade" + ) + + # Update config with OCA_OpenUpgrade + ignore_path = ( + "--ignore-odoo-path " if next_version <= 13 else "" + ) + extra_addons_path_extra = ( + f",{path_addons_openupgrade}/addons,{path_addons_openupgrade}/odoo/addons" + if next_version <= 13 + else "" + ) + cmd_update_config = ( + f"./script/git/git_repo_update_group.py {ignore_path}" + f"--extra-addons-path {path_addons_openupgrade}{extra_addons_path_extra} " + f"&& ./script/generate_config.sh" + ) + self.todo_upgrade_execute(cmd_update_config) + + print("🚸 Please, validate commits after code migration.") + print("ℹ To show repo status :\nmake repo_show_status") + print( + f"🚸 Please, validate this path into config.conf : '{path_addons_openupgrade}'." + ) + status = input(f"💬 Press to continue {msg} : ").strip() + # The technique change at version 14 + if next_version <= 13: + erplibre_version = self.install_OCA_openupgrade( + next_version + ) + cmd_upgrade = f".venv.{erplibre_version}/bin/python ./odoo{next_version}.0/OCA_OpenUpgrade/odoo-bin -c ./config.conf --update all --no-http --stop-after-init -d {database_name_upgrade}" + else: + cmd_upgrade = f"./run.sh --upgrade-path=./odoo{next_version}.0/OCA_OpenUpgrade/openupgrade_scripts/scripts --update all -c config.conf --stop-after-init --no-http --load=base,web,openupgrade_framework -d {database_name_upgrade}" + lst_upgrade_odoo[index] = cmd_upgrade + + self.todo_upgrade_execute( + cmd_upgrade, + new_env={ + "OPENUPGRADE_TARGET_VERSION": f"{next_version}.0" + }, + ) + # TODO detect error + + self.dct_progression["state_4_upgrade_odoo_lst"] = ( + lst_upgrade_odoo + ) + self.write_config() + + str_wait_next_version = ( + " (or wait next version 🤖)" + if next_version != lst_next_version[-1] + else "" + ) + + status = ( + input( + f"💬 Do you want to upgrade all{str_wait_next_version}? Press y/Y to upgrade all addons database : " + ) + .strip() + .lower() + ) + + if status == "y": + self.todo_upgrade_execute( + f"./script/addons/update_addons_all.sh {database_name_upgrade}", + ) + + print(f"✅ -> Database upgrade Odoo{next_version} done") + + # Update config without OCA_OpenUpgrade + cmd_update_config = f"./script/git/git_repo_update_group.py && ./script/generate_config.sh" + self.todo_upgrade_execute(cmd_update_config) + print("[y] Open server with Selenium") + status = ( + input( + "💬 Do you want to test this upgrade? Choose or press to ignore it : " + ) + .strip() + .lower() + ) + "make repo_show_status" + if status == "y": + self.todo.prompt_execute_selenium_and_run_db( + database_name_upgrade + ) + status = input( + f"💬 Press to continue 4.{index} : " + ).strip() + else: + print(f"✅ -> Database upgrade Odoo{next_version} - nothing") + + # + # waiting_input = input("💬 Press any keyboard key to continue...") + print("") + + msg = "5 - Cleaning up database after upgrade" + print(f"🔷 {msg}") + self.add_comment_progression(msg) + + print( + "✨ Re-update i18n, purger data, tables (except mail_test and mail_test_full)" + ) + # waiting_input = input("💬print Press any keyboard key to continue...") + msg = "6 - Migration finished" + print(f"🔷 {msg}") + self.add_comment_progression(msg) + + cmd_backup_template = f"./odoo_bin.sh db --backup --database {database_name_upgrade} --restore_image" + cmd_backup = f"{cmd_backup_template} {database_name_upgrade}_finish_{datetime.datetime.now().strftime('%Y%m%d%H%M%S')}" + print(f"✨ Can execute backup creation :\n{cmd_backup}") + status = input( + "💬 Press y/Y or write filename.zip to export or enter to continue : " + ).strip() + if status.lower(): + if status.lower() != "y": + cmd_backup = f"{cmd_backup_template} {status}" + self.todo_upgrade_execute(cmd_backup) + + status = input("💬 Test the migration, press y/Y : ") + if status.lower().strip() == "y": + self.todo.prompt_execute_selenium_and_run_db(database_name_upgrade) + + def get_rename_module(self, lst_module, next_version): + path_search = f"odoo{next_version}.0/OCA_OpenUpgrade/" + status, cmd_executed, lst_output = self.todo_upgrade_execute( + f"find {path_search} -name apriori.py", + get_output=True, + ) + + if not lst_output: + _logger.error( + f"Cannot find renamed module script apriori.py into path '{path_search}'" + ) + return lst_module + apriory_py = lst_output[0].strip() + + with open(apriory_py, "r") as f: + file_content = f.read() + + data_vars = {} + exec(file_content, data_vars) + renamed_modules = data_vars.get("renamed_modules", {}) + merged_modules = data_vars.get("merged_modules", {}) + deleted_modules = data_vars.get("deleted_modules", []) + + lst_index_to_delete = [] + for index, module in enumerate(lst_module): + renamed_module = renamed_modules.get(module) + merged_module = merged_modules.get(module) + if renamed_module: + lst_module[index] = renamed_module + if merged_module: + lst_module[index] = merged_module + if module in deleted_modules: + lst_index_to_delete.append(index) + for index_to_delete in lst_index_to_delete[::-1]: + lst_module.pop(index_to_delete) + return list(set(lst_module)) + + def search_module_to_move(self, source_version_odoo, target_version_odoo): + lst_module = [] + # lst_path_to_check = [ + # os.path.join(f"odoo{actual_version_odoo}", "addons"), + # os.path.join(f"odoo{target_version_odoo}", "addons"), + # os.path.join(f"private", "addons"), + # ] + source_path_to_check = os.path.join( + f"odoo{source_version_odoo}.0", "addons", "addons" + ) + target_path_to_check = os.path.join( + f"odoo{target_version_odoo}.0", "addons", "addons" + ) + + # Search + is_moving_git = False + if os.path.exists(source_path_to_check): + if os.path.exists(os.path.join(source_path_to_check, ".git")): + is_moving_git = True + if not os.path.exists(target_path_to_check): + shutil.copytree(source_path_to_check, target_path_to_check) + # if not is_moving_git: + # os.mkdir(path_to_check_target) + # else: + # # TODO clone + # pass + + if not is_moving_git: + # TODO do something + # Time to compare + os.listdir(source_path_to_check) + + if os.path.exists(target_path_to_check): + for dir_name in os.listdir(target_path_to_check): + source_module_path = os.path.join( + source_path_to_check, dir_name + ) + source_manifest_path = os.path.join( + source_module_path, "__manifest__.py" + ) + + target_module_path = os.path.join( + target_path_to_check, dir_name + ) + target_manifest_path = os.path.join( + target_module_path, "__manifest__.py" + ) + if os.path.exists(target_manifest_path): + # TODO remove from list when module already exist in version 15 + dct_module = { + "source_module_path": source_module_path, + "source_manifest_path": source_manifest_path, + "source_addons_path": source_path_to_check, + "target_module_path": target_module_path, + "target_manifest_path": target_manifest_path, + "target_addons_path": target_path_to_check, + "module_name": dir_name, + "source_version_odoo": source_version_odoo, + "target_version_odoo": target_version_odoo, + } + lst_module.append(dct_module) + + dct_module = { + "lst_module": lst_module, + "source_module_path": source_path_to_check, + "target_module_path": target_path_to_check, + } + return dct_module + + def uninstall_from_database( + self, lst_module_to_uninstall, database_name, actual_version + ): + if not lst_module_to_uninstall: + return + uninstall_module = ",".join(lst_module_to_uninstall) + self.todo_upgrade_execute( + f"./script/addons/uninstall_addons.sh {database_name} {uninstall_module}", + single_source_odoo=True, + ) + + # Update list installed module + self.dct_module_per_version[actual_version] = sorted( + list( + set(self.dct_module_per_version[actual_version]) + - set(lst_module_to_uninstall) + ) + ) + self.dct_progression["dct_module_per_version"] = ( + self.dct_module_per_version + ) + self.write_config() + + def install_from_database( + self, lst_module_to_install, database_name, actual_version + ): + if not lst_module_to_install: + return + install_module = ",".join(lst_module_to_install) + self.todo_upgrade_execute( + f"./script/addons/install_addons.sh {database_name} {install_module}", + single_source_odoo=True, + ) + + # Update list installed module + self.dct_module_per_version[actual_version] = sorted( + list( + set( + self.dct_module_per_version[actual_version] + + lst_module_to_install + ) + ) + ) + self.dct_progression["dct_module_per_version"] = ( + self.dct_module_per_version + ) + self.write_config() + + def check_addons_exist( + self, lst_module_to_check, ignore_error=True, get_all_info=False + ): + str_module_to_check = ",".join(sorted(lst_module_to_check)) + status, cmd_executed, dct_output = self.todo_upgrade_execute( + f"{PYTHON_BIN} ./script/addons/check_addons_exist.py --format_json --output_json -m {str_module_to_check}", + get_output=True, + output_is_json=True, + wait_at_error=not ignore_error, + ) + + lst_module_missing = dct_output.get("missing") + lst_module_duplicate = dct_output.get("duplicate") + if get_all_info: + lst_module_error = dct_output.get("error") + lst_module_exist = dct_output.get("exist") + return ( + lst_module_missing, + lst_module_duplicate, + lst_module_exist, + lst_module_error, + ) + + return lst_module_missing, lst_module_duplicate + + def switch_odoo(self, odoo_version): + int_odoo_version = int(float(odoo_version)) + + # Expect odoo_version like 12.0 + lst_version, lst_version_installed, odoo_installed_version = ( + self.todo.get_odoo_version() + ) + if odoo_installed_version != f"odoo{int_odoo_version}.0": + print( + f"⧖ -> Was '{odoo_installed_version}', Switch to odoo{int_odoo_version}.0" + ) + self.todo_upgrade_execute(f"make switch_odoo_{int_odoo_version}") + self.todo_upgrade_execute("make config_gen_all") + + def install_OCA_odoo_module_migrator(self): + if not os.path.exists(PATH_VENV_MODULE_MIGRATOR): + self.todo_upgrade_execute( + f"cd {PATH_OCA_ODOO_MODULE_MIGRATOR} && python -m venv {VENV_NAME_MODULE_MIGRATOR} && source {VENV_NAME_MODULE_MIGRATOR}/bin/activate && pip3 install -r requirements.txt" + ) + + def install_OCA_openupgrade(self, next_version): + # TODO install odoorpc==0.7.0 + # openupgradelib + # openupgrade_path = f"odoo{next_version}.0/OCA_OpenUpgrade" + # venv_oca_path = f"{openupgrade_path}/.venv" + # if os.path.exists(venv_oca_path): + # return + lst_version, lst_version_installed, odoo_installed_version = ( + self.todo.get_odoo_version() + ) + extract_version = f"{next_version}.0" + dct_erplibre_info = [ + a for a in lst_version if a.get("odoo_version") == extract_version + ] + if not dct_erplibre_info: + raise Exception(f"Cannot extract {extract_version}") + dct_erplibre_info = dct_erplibre_info[0] + erplibre_version = dct_erplibre_info.get("erplibre_version") + # self.todo_upgrade_execute( + # f".venv.{erplibre_version}/bin/python -m venv {venv_oca_path} && {venv_oca_path}/bin/pip3 install -r {openupgrade_path}/requirements.txt" + # ) + self.todo_upgrade_execute( + f".venv.{erplibre_version}/bin/pip install odoorpc==0.7.0" + ) + self.todo_upgrade_execute( + f".venv.{erplibre_version}/bin/pip install openupgradelib" + ) + return erplibre_version + + def todo_upgrade_execute( + self, + cmd, + single_source_odoo=False, + new_env=None, + quiet=False, + get_output=False, + output_is_json=False, + wait_at_error=True, + ): + if output_is_json and not get_output: + get_output = True + output = None + if get_output: + status, cmd_executed, output = self.todo.executer_commande_live( + cmd, + source_erplibre=False, + single_source_odoo=single_source_odoo, + new_env=new_env, + return_status_and_output_and_command=True, + quiet=quiet, + ) + else: + status, cmd_executed = self.todo.executer_commande_live( + cmd, + source_erplibre=False, + single_source_odoo=single_source_odoo, + new_env=new_env, + return_status_and_command=True, + quiet=quiet, + ) + self.lst_command_executed.append(cmd_executed) + self.dct_progression["command_executed"] = self.lst_command_executed + self.write_config() + if status and wait_at_error: + print("[1] to redo the command") + wait_status = ( + input( + "💬 Error detected, press to continue or ctrl+c to stop : " + ) + .strip() + .lower() + ) + if wait_status == "1": + return self.todo_upgrade_execute( + cmd, + single_source_odoo=single_source_odoo, + new_env=new_env, + quiet=quiet, + get_output=get_output, + output_is_json=output_is_json, + wait_at_error=wait_at_error, + ) + + if get_output: + if output_is_json: + str_output = json.loads("".join(output)) + return status, cmd_executed, str_output + return status, cmd_executed, output + return status, cmd_executed + + def check_and_clone_source_to_target_migration_code( + self, next_version, source_addons_path, target_addons_path + ): + if not os.path.exists(os.path.join(source_addons_path, ".git")): + return + if os.path.exists(os.path.join(target_addons_path, ".git")): + return + # if not os.path.exists(target_addons_path): + # cmd_mkdir = f"mkdir -p {target_addons_path}" + # status, cmd_executed, lst_output = self.todo_upgrade_execute( + # cmd_mkdir, + # get_output=True, + # ) + # return + source_dir_name = os.path.basename(source_addons_path) + # Clone a project for next version + # Get actual branch + cmd_git_clone_migrate_source = ( + f"cd {source_addons_path} && " + f"git branch --show-current && " + f"cd ~-" + ) + status, cmd_executed, lst_output = self.todo_upgrade_execute( + cmd_git_clone_migrate_source, + get_output=True, + ) + branch_source = lst_output[0].strip() if len(lst_output) else "" + if not branch_source: + # Get branch from repo + branch_source = self.get_branch_name_from_local_manifest( + source_addons_path, f"{next_version}.0" + ) + branch_target = branch_source.replace( + str(next_version - 1), str(next_version) + ) + # Get remote branch for actual version + cmd_git_clone_migrate_source_same_target = ( + f"cd {source_addons_path} " + f"&& git fetch --all " + f'&& git branch -vv | grep "{branch_source}" ' + f"&& cd ~-" + ) + cmd_git_clone_migrate_source_same_target_remote_only = ( + f"cd {source_addons_path} && git remote && cd ~-" + ) + status, cmd_executed, lst_output = self.todo_upgrade_execute( + cmd_git_clone_migrate_source_same_target, + get_output=True, + wait_at_error=False, + ) + if status == 1: + status, cmd_executed, lst_output = self.todo_upgrade_execute( + cmd_git_clone_migrate_source_same_target_remote_only, + get_output=True, + ) + if lst_output: + remote = lst_output[0].strip() + # TODO write this modification into repo manifest + else: + remote = input( + "👹 BUG, you need to push last branch. Please write the remote/ :" + ) + else: + local_branch, remote, remote_branch = ( + self.get_local_branch_remote_actual_branch_git( + lst_output, + ) + ) + # Get remote branch for next version + remote_branch_target = f"{remote}/{branch_target}" + cmd_git_clone_migrate_source_same_target = ( + f"cd {source_addons_path} " + f"&& git fetch --all " + f'&& git branch --remotes -vv | grep "{remote_branch_target} " ' + f"&& cd ~-" + ) + status, cmd_executed, lst_output = self.todo_upgrade_execute( + cmd_git_clone_migrate_source_same_target, + get_output=True, + wait_at_error=False, + ) + has_existing_target_branch = any([a.strip() for a in lst_output]) + + # TODO check config if path is added + # Get remote branch address + cmd_remote_address = ( + f"cd {source_addons_path} " + f"&& git remote get-url {remote} " + f"&& cd ~-" + ) + status, cmd_executed, lst_output = self.todo_upgrade_execute( + cmd_remote_address, + get_output=True, + ) + + remote_address = lst_output[0].strip() + + # TODO some time, the clone has error, need to repeat + branch_to_clone = ( + branch_target if has_existing_target_branch else branch_source + ) + + cmd_git_clone = ( + f"cd {os.path.dirname(target_addons_path)} " + f"&& git clone {remote_address} {source_dir_name} -b {branch_to_clone} && cd ~-" + ) + status, cmd_executed, lst_output = self.todo_upgrade_execute( + cmd_git_clone, + get_output=True, + ) + if not has_existing_target_branch: + cmd_git_clone = ( + f"cd {target_addons_path} " + f"&& git checkout -b {branch_target} && cd ~-" + ) + status, cmd_executed, lst_output = self.todo_upgrade_execute( + cmd_git_clone, + get_output=True, + ) + + def get_branch_name_from_local_manifest(self, addons_path, default_branch): + relative_addons_path = addons_path.replace(os.getcwd() + "/", "") + git_tool = GitTool() + dct_remote, dct_project, default_remote = ( + git_tool.get_manifest_xml_info(filename=LOCAL_MANIFEST) + ) + for dct_repo in dct_project.values(): + path = dct_repo.get("@path") + if path == relative_addons_path: + revision = dct_repo.get("@revision") + return revision + return default_branch + + def get_local_branch_remote_actual_branch_git(self, lst_output): + for line in lst_output: + # The current branch is marked with an asterisk (*) at the start of the line + if not line.strip().startswith("*"): + continue + # Split the line to isolate the remote and remote branch name + parts = line.split() + if len(parts) >= 4: + # The remote and remote branch are in the 4th part, e.g., '[origin/main]' + remote_info = parts[3].strip("[]") + # Split 'origin/main' into 'origin' and 'main' + # remote, remote_branch = remote_info.split("/", 1) + try: + remote, remote_branch = remote_info.split("/", 1) + except ValueError as e: + # # TODO this means no remote, take default one? or last one? + # # TODO supporter la migration 17 vers 18 + # print("👹 BUG, you need to push ") + # TODO search remote and associate branch + value = input( + "👹 BUG, you need to push last branch. Please write the remote/branch_name :" + ) + remote, remote_branch = value.split("/", 1) + return parts[1], remote, remote_branch + return None, None, None + + def add_comment_progression(self, comment): + comment_to_add = f"# {comment}" + self.lst_command_executed.append(comment_to_add) + self.dct_progression["command_executed"] = self.lst_command_executed + self.write_config()