erplibre/script/database/image_db.py

#!/usr/bin/env python3
# © 2021-2026 TechnoLibre (http://www.technolibre.ca)
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl)

import argparse
import json
import logging
import os
import subprocess
import sys
import time
import uuid
from collections import defaultdict

logging.basicConfig(level=os.environ.get("LOGLEVEL", "INFO"))

_logger = logging.getLogger(__name__)

PYTHON_BIN = ".venv.erplibre/bin/python3"
IMAGE_DB_BIN = f"{PYTHON_BIN} ./script/database/image_db.py"


def get_config():
    """Parse command line arguments, extracting the config file name,
    returning the union of config file and command line arguments

    :return: dict of config file settings and command line arguments
    """
    # TODO update description
    parser = argparse.ArgumentParser(
        formatter_class=argparse.RawDescriptionHelpFormatter,
        description="""\
DESCRIPTION
    Create image for database restoration to increase speed. Check file

SUGGESTION
    ./script/database/image_db.py --image odoo16.0_base --odoo_version 16.0
""",
        epilog="""\
""",
    )
    parser.add_argument(
        "--image",
        help=(
            "Image name to restore, from directory image_db, filename without"
            " '.zip'. Example, use odoo12.0_base to use image"
            " odoo12.0_base.zip. Default value is odoo12.0_base"
        ),
    )
    parser.add_argument(
        "--odoo_version",
        help=(
            "Force Odoo version, example 12.0 or 16.0, if empty, use from"
            " .odoo-version file"
        ),
    )
    parser.add_argument(
        "--generate_list_only",
        action="store_true",
        help=(
            "Generate list command in bash to run all package from"
            " odoo_version"
        ),
    )
    parser.add_argument(
        "--generate_bash_cmd_parallel",
        action="store_true",
        help=(
            "Generate list with parallel command in bash to run all package"
            " from odoo_version"
        ),
    )
    parser.add_argument(
        "--show_list_only",
        action="store_true",
        help="Show list of all package about Odoo version",
    )
    parser.add_argument(
        "--check_addons_exist",
        action="store_true",
        help=(
            "Will return an error and stop execution if detect a non existing"
            " addons from actual odoo version."
        ),
    )
    parser.add_argument(
        "--keep_database",
        action="store_true",
        help="Keep temporary database",
    )
    args = parser.parse_args()
    return args


def main():
    config = get_config()
    if not config.odoo_version:
        # Fill it from system
        filename_odoo_version = ".odoo-version"
        if not os.path.isfile(filename_odoo_version):
            _logger.error(f"Missing file {filename_odoo_version}")
            sys.exit(1)
        with open(filename_odoo_version, "r") as f:
            config.odoo_version = f.readline()

    # Open configuration file
    config_file = os.path.join("conf", f"module_list_image_db_odoo.json")
    if not os.path.exists(config_file):
        _logger.error(f"Configuration file {config_file} does not exist.")
        sys.exit(1)

    with open(config_file, "r") as f:
        dct_config_all_image = json.load(f)

    odoo_prefix_version = f"odoo{config.odoo_version}"

    if config.show_list_only:
        lst_image_to_show = set()
        for image_db_name, dct_image_db in dct_config_all_image.items():
            if not image_db_name.startswith(
                odoo_prefix_version
            ) or dct_image_db.get("disable"):
                continue
            # First always need to be a base
            image_list = dct_image_db.get("image_list")
            for dct_image in image_list:
                pkg_name = dct_image.get("pkg_name")
                if pkg_name:
                    sub_image_db_name = f"{image_db_name}_{pkg_name}"
                else:
                    sub_image_db_name = image_db_name
                lst_image_to_show.add(sub_image_db_name)

        lst_to_show = sorted(lst_image_to_show)
        for image_to_show in lst_to_show:
            print(image_to_show)
        sys.exit(0)

    if config.generate_list_only:
        for image_db_name, dct_image_db in dct_config_all_image.items():
            if not image_db_name.startswith(
                odoo_prefix_version
            ) or dct_image_db.get("disable"):
                continue
            # First always need to be a base
            print(
                f"{IMAGE_DB_BIN} --odoo_version"
                f" {config.odoo_version} --image {image_db_name}"
            )
        sys.exit(0)

    if config.check_addons_exist:
        lst_module_to_check = set()
        for image_db_name, dct_image_db in dct_config_all_image.items():
            if not image_db_name.startswith(
                odoo_prefix_version
            ) or dct_image_db.get("disable"):
                continue
            lst_image_list = dct_image_db.get("image_list")
            for dct_image_config in lst_image_list:
                lst_module = dct_image_config.get("module")
                lst_module_to_check.update(lst_module)
        lst_module_missing = []
        for module_to_check in lst_module_to_check:
            cmd_check = (
                f"{PYTHON_BIN} ./script/addons/check_addons_exist.py -m"
                f" {module_to_check}"
            )
            status = run_cmd(cmd_check, quiet=True, sys_exit=False)
            if status:
                lst_module_missing.append(module_to_check)
        if lst_module_missing:
            print(f"Missing module :")
            print(sorted(lst_module_missing))
            sys.exit(1)
        else:
            print("No module missing")
        sys.exit(0)

    if config.generate_bash_cmd_parallel:
        dct_depend_image = defaultdict(list)
        lst_total_image = []

        lst_distribute_image = []
        lst_queue_parallel = []
        dct_image_delay = defaultdict(int)
        # Search dependency
        for image_db_name, dct_image_db in dct_config_all_image.items():
            if not image_db_name.startswith(
                odoo_prefix_version
            ) or dct_image_db.get("disable"):
                continue
            if image_db_name in lst_total_image:
                _logger.error(f"Duplicate image_db_name: {image_db_name}")
                sys.exit(1)
            base_name = dct_image_db.get("base", "")
            dct_depend_image[base_name].append(image_db_name)
            lst_total_image.append(image_db_name)
            dct_image_delay[image_db_name] = dct_image_db.get("delay", 0)

        # Reorder it
        max_iter = 1000
        i = 0
        while dct_depend_image and i < max_iter:
            i += 1

            if "" in dct_depend_image.keys():
                # Get from empty key for root dependency
                lst_module = dct_depend_image.get("")
                lst_queue_parallel.append(lst_module)
                lst_distribute_image = lst_module[:]
                del dct_depend_image[""]
            else:
                # Search if dependency already into list
                lst_module = []
                lst_key_to_delete = []
                lst_distribute_image_to_append = []
                for key_to_check, lst_value in dct_depend_image.items():
                    if key_to_check in lst_distribute_image:
                        # Dependency find
                        # Search for delay
                        has_delay = False
                        for value_module in lst_value:
                            delay = dct_image_delay.get(value_module)
                            if delay:
                                has_delay = True
                                dct_image_delay[value_module] = delay - 1
                            elif value_module not in lst_distribute_image:
                                lst_distribute_image_to_append.append(
                                    value_module
                                )
                                lst_module.append(value_module)
                        if not has_delay:
                            lst_key_to_delete.append(key_to_check)
                lst_distribute_image.extend(lst_distribute_image_to_append)
                if lst_module:
                    lst_queue_parallel.append(lst_module)
                for key_to_delete in lst_key_to_delete:
                    del dct_depend_image[key_to_delete]

        # not finish to empty the queue
        if dct_depend_image:
            _logger.info(
                "Missing dependencies, auto-run all image generation for last"
                " execution"
            )
            for depend_image, lst_module in dct_depend_image.items():
                lst_queue_parallel.append(lst_module)
        # print command to execute
        lst_cmd = []
        for lst_mod in lst_queue_parallel:
            if not lst_mod:
                continue
            str_module = " ".join(lst_mod)
            cmd = f"echo 'Generate imageDB : [{str_module}]'\n"
            if len(lst_mod) > 1:
                cmd += "parallel ::: " + " ".join(
                    [
                        f'"{IMAGE_DB_BIN} --odoo_version'
                        f' {config.odoo_version} --image {a}"'
                        for a in lst_mod
                    ]
                )
            else:
                cmd += (
                    f"{IMAGE_DB_BIN} --odoo_version"
                    f" {config.odoo_version} --image {lst_mod[0]}"
                )
            cmd += "\nstatus=$?; [ $status -ne 0 ] && exit $status"

            lst_cmd.append(cmd)
        print("\n".join(lst_cmd))
        sys.exit(0)

    image_name_to_generate = (
        config.image if config.image else f"{odoo_prefix_version}_base"
    )
    dct_config_image = dct_config_all_image.get(image_name_to_generate)

    if not dct_config_image:
        _logger.error(f"Cannot retrieve image name '{image_name_to_generate}'")
        sys.exit(1)

    if dct_config_image.get("disable"):
        _logger.info("Ignore this image DB generation, because it's disabled.")
        sys.exit(1)

    bd_temp_name = (
        f"temp_img_create_{image_name_to_generate}_{uuid.uuid4().hex[:6]}"
    )

    # Summary
    # Step 0, drop and restore
    # Step 1, install addons
    # Step 2, uninstall if need it
    # Step 3, install theme
    # Step 4, create image_db by backup
    # Step 5, loop to 1 with same bd name

    base_image_name = dct_config_image.get("base")
    all_temp_bd = []

    # Step 0, drop and restore
    cmd_drop_db = f"./odoo_bin.sh db --drop --database {bd_temp_name}"
    all_temp_bd.append(bd_temp_name)
    run_cmd(cmd_drop_db)
    if not base_image_name or base_image_name == image_name_to_generate:
        with_demo = dct_config_image.get("with_demo")
        # Create a new one
        cmd = f"./odoo_bin.sh db --create --database" f" {bd_temp_name}"
        if with_demo:
            cmd += " --demo"
    else:
        cmd = (
            "./script/database/db_restore.py --database"
            f" {bd_temp_name} --image {base_image_name}"
        )
    run_cmd(cmd)
    for dct_value in dct_config_image.get("image_list"):
        pkg_name = dct_value.get("pkg_name", "")
        # Step 1, install addons
        lst_module = ",".join(dct_value.get("module"))
        cmd = f"./script/addons/install_addons.sh {bd_temp_name} {lst_module}"
        run_cmd(cmd)
        # Step 2, uninstall if need it
        # Step 3, install theme
        module_theme = dct_value.get("theme")
        if module_theme:
            cmd = (
                "./script/addons/install_addons_theme.sh"
                f" {bd_temp_name} {module_theme}"
            )
            run_cmd(cmd)
        # Step 4, create image_db by backup
        module_image_name = (
            image_name_to_generate
            if not pkg_name
            else f"{image_name_to_generate}_{pkg_name}"
        )
        cmd = (
            f"./odoo_bin.sh db --backup --database"
            f" {bd_temp_name} --restore_image {module_image_name}"
        )
        run_cmd(cmd)
        # Step 5, loop to 1 with same bd name for next package

    # Step 6, clean if ask
    if not config.keep_database:
        for db_name in all_temp_bd:
            cmd_drop_db = f"./odoo_bin.sh db --drop --database {db_name}"
            run_cmd(cmd_drop_db)


def run_cmd(cmd, quiet=False, sys_exit=True):
    # status = os.system(cmd)
    # if status != 0:
    #     _logger.error(f"Command failed : '{cmd}'")
    #     sys.exit(status)
    if not quiet:
        _logger.info(f"Run cmd: {cmd}")
    debut = time.time()
    # Maybe can use exec_command_live
    process = subprocess.Popen(
        cmd,
        shell=True,
        text=True,
        stdout=subprocess.PIPE,
        stderr=subprocess.PIPE,
    )
    stdout, stderr = process.communicate()
    fin = time.time()
    status_code = process.returncode
    if stderr and not quiet:
        _logger.error(stderr)
    execution_time = fin - debut
    if not quiet:
        _logger.info(f"Time execution: {execution_time:.2f} seconds\n")
    if status_code != 0:
        if not quiet:
            _logger.error(f"Command failed : '{cmd}'")
        if sys_exit:
            sys.exit(status_code)
    return status_code


if __name__ == "__main__":
    main()