Merge branch 'makefile_docker_migration'

- fix migration to version 1.2.0
- add script to create image backup
- add script to show git repo diff
- better documentation

CHANGELOG.md

@@ -7,6 +7,18 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm
 
 ## [Unreleased]
 
+**Migration note**
+
+Because addons repository has change, config file need to be updated.
+- When upgrading to version 1.2.0:
+  - From docker
+    - Run script `make docker_exec_erplibre_gen_config`
+    - Restart the docker `make docker_restart_daemon`
+  - From vanilla
+    - Run script `make install_dev`
+    - Restart your daemon
+    - Regenerate master password manually
+
 ### Added
 
 - Adapt script to give an execution status
@@ -15,8 +27,18 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm
 - Script to check git diff and ignore date
 - Repo with ERPLibre image
 - Improve git repo usage, filter repo by use case
+- ERPLibre theme website of TechnoLibre
+- ERPLibre website snippet
+  - Basic HTML snippets
+  - Snippet card
+  - Snippet timelines
+- Module contract_digitized_signature with contract_portal
+- Module disable auto_backup
+- Odoo cli db command to manipulate restoration db
+- Odoo cli i18n command to generate i18n pot files
 
 #### Makefile
+
 - Format code
 - Code generator test
 - Addons installation
@@ -25,8 +47,9 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm
 - Docker execution
 
 #### Code generator
+
 - Code generator for Odoo module, depending of ERPLibre
-- Support map geoengine
+- Support map geospatial
 - Support i18n
 - Script to transform Python and XML to Python code writer script to regenerate themselves
 
@@ -34,6 +57,9 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm
 
 - Update Python dependency with Poetry
 - Format all Python code with black
+- Module auto_backup with sftp host key
+- Module muk_website_branding use ERPLibre branding
+- Update docs with vscode support, custom document layout, custom email template and trick to use params to share variable
 
 #### Docker
 
@@ -45,6 +71,7 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm
 
 - Ubuntu installation
 - Poetry installation
+- Geospatial with postgis can be installed
 
 ## [1.1.1] - 2020-12-11
 

Makefile

@@ -107,6 +107,10 @@ db_drop_db_code_generator:
 db_drop_db_template:
 	./.venv/bin/python3 ./odoo/odoo-bin db --drop --database template
 
+.PHONY: db_clean_cache
+db_clean_cache:
+	./script/db_restore.py --clean_cache
+
 .PHONY: db_restore_erplibre_base_db_test
 db_restore_erplibre_base_db_test:
 	./script/db_restore.py --database test
@@ -143,6 +147,66 @@ db_restore_erplibre_base_db_code_generator:
 db_restore_erplibre_base_db_template:
 	./script/db_restore.py --database template
 
+.PHONY: db_create_db_test
+db_create_db_test: db_drop_db_test
+	./.venv/bin/python3 ./odoo/odoo-bin db --create --database test
+
+########################
+#  Image installation  #
+########################
+.PHONY: image_db_create_erplibre_base
+image_db_create_erplibre_base:
+	./script/make.sh db_create_db_test
+	./script/addons/install_addons.sh test erplibre_base
+	./.venv/bin/python3 ./odoo/odoo-bin db --backup --database test --restore_image erplibre_base
+
+.PHONY: image_db_create_erplibre_website
+image_db_create_erplibre_website:
+	./script/make.sh db_create_db_test
+	./script/addons/install_addons.sh test erplibre_base,website,erplibre_website_snippets_basic_html,erplibre_website_snippets_cards,erplibre_website_snippets_structures,erplibre_website_snippets_timelines,website_form_builder
+	./.venv/bin/python3 ./odoo/odoo-bin db --backup --database test --restore_image erplibre_website
+	./script/addons/install_addons.sh test crm,website_crm
+	./.venv/bin/python3 ./odoo/odoo-bin db --backup --database test --restore_image erplibre_website_crm
+	./script/addons/install_addons.sh test website_livechat
+	./.venv/bin/python3 ./odoo/odoo-bin db --backup --database test --restore_image erplibre_website_chat_crm
+	./script/addons/install_addons.sh test website_sale,erplibre_base_quebec
+	./.venv/bin/python3 ./odoo/odoo-bin db --backup --database test --restore_image erplibre_ecommerce_base
+	./script/addons/install_addons.sh test stock,purchase,website_sale_management
+	./.venv/bin/python3 ./odoo/odoo-bin db --backup --database test --restore_image erplibre_ecommerce_advance
+	./script/addons/install_addons.sh test project
+	./.venv/bin/python3 ./odoo/odoo-bin db --backup --database test --restore_image erplibre_ecommerce_project
+	./script/addons/install_addons.sh test pos_sale
+	./.venv/bin/python3 ./odoo/odoo-bin db --backup --database test --restore_image erplibre_ecommerce_pos
+	./script/addons/install_addons.sh test hr
+	./.venv/bin/python3 ./odoo/odoo-bin db --backup --database test --restore_image erplibre_ecommerce_pos_hr
+
+.PHONY: image_db_create_all
+image_db_create_all:
+	./script/make.sh db_create_db_test
+	./script/addons/install_addons.sh test erplibre_base
+	./.venv/bin/python3 ./odoo/odoo-bin db --backup --database test --restore_image erplibre_base
+	./script/addons/install_addons.sh test website,erplibre_website_snippets_basic_html,erplibre_website_snippets_cards,erplibre_website_snippets_structures,erplibre_website_snippets_timelines,website_form_builder
+	./.venv/bin/python3 ./odoo/odoo-bin db --backup --database test --restore_image erplibre_website
+	./script/addons/install_addons.sh test crm,website_crm
+	./.venv/bin/python3 ./odoo/odoo-bin db --backup --database test --restore_image erplibre_website_crm
+	./script/addons/install_addons.sh test website_livechat
+	./.venv/bin/python3 ./odoo/odoo-bin db --backup --database test --restore_image erplibre_website_chat_crm
+	./script/addons/install_addons.sh test website_sale,erplibre_base_quebec
+	./.venv/bin/python3 ./odoo/odoo-bin db --backup --database test --restore_image erplibre_ecommerce_base
+	./script/addons/install_addons.sh test stock,purchase,website_sale_management
+	./.venv/bin/python3 ./odoo/odoo-bin db --backup --database test --restore_image erplibre_ecommerce_advance
+	./script/addons/install_addons.sh test project
+	./.venv/bin/python3 ./odoo/odoo-bin db --backup --database test --restore_image erplibre_ecommerce_project
+	./script/addons/install_addons.sh test pos_sale
+	./.venv/bin/python3 ./odoo/odoo-bin db --backup --database test --restore_image erplibre_ecommerce_pos
+	./script/addons/install_addons.sh test hr
+	./.venv/bin/python3 ./odoo/odoo-bin db --backup --database test --restore_image erplibre_ecommerce_pos_hr
+
+.PHONY: image_diff_base_website
+image_diff_base_website:
+	#./script/manifest/compare_backup.py --backup_file_1 ./image_db/erplibre_base.zip --backup_file_2 ./image_db/erplibre_website.zip
+	./script/manifest/compare_backup.py --backup_1 erplibre_base --backup_2 erplibre_website
+
 #########################
 #  Addons installation  #
 #########################
@@ -241,6 +305,13 @@ test_code_generator_code_i18n_extra:
 	# To support i18n in auto_backup
 	./script/code_generator/check_git_change_code_generator.sh ./addons/OCA_server-tools/auto_backup
 
+##############
+#  tag  #
+##############
+.PHONY: tag_push_all
+tag_push_all:
+	./script/tag_push_all.py
+
 ##############
 #  terminal  #
 ##############
@@ -295,6 +366,9 @@ docker_run_daemon:
 docker_stop:
 	docker-compose down
 
+.PHONY: docker_restart_daemon
+docker_restart_daemon: docker_stop docker_run_daemon
+
 .PHONY: docker_show_logs_live
 docker_show_logs_live:
 	docker-compose logs -f
@@ -305,17 +379,29 @@ docker_show_process:
 
 .PHONY: docker_exec_erplibre
 docker_exec_erplibre:
-	docker exec -u root -ti erplibre_ERPLibre_1 bash
+	./script/docker/docker_exec.sh
+
+.PHONY: docker_exec_erplibre_gen_config
+docker_exec_erplibre_gen_config:
+	./script/docker/docker_gen_config.sh
+
+.PHONY: docker_exec_erplibre_make_test
+docker_exec_erplibre_make_test:
+	./script/docker/docker_make_test.sh
+
+.PHONY: docker_exec_erplibre_repo_show_status
+docker_exec_erplibre_repo_show_status:
+	./script/docker/docker_repo_show_status.sh
 
 # build docker
 .PHONY: docker_build
 docker_build:
-	./script/docker_build.sh
+	./script/docker/docker_build.sh
 
 # build docker release
 .PHONY: docker_build_release
 docker_build_release:
-	./script/docker_build.sh --release
+	./script/docker/docker_build.sh --release
 
 # docker clean all
 .PHONY: docker_clean_all
@@ -340,6 +426,26 @@ repo_configure_all:
 repo_configure_group_code_generator:
 	./script/update_manifest_local_dev_code_generator.sh
 
+# Show git status for all repo
+.PHONY: repo_show_status
+repo_show_status:
+	./.venv/repo forall -pc "git status -s"
+
+# Show divergence between actual repository and production manifest
+.PHONY: repo_diff_manifest_production
+repo_diff_manifest_production:
+	./script/git_show_code_diff_repo_manifest.py
+
+# Show git diff for all repo from last tag version release
+.PHONY: repo_diff_from_last_version
+repo_diff_from_last_version:
+	./script/repo_diff_last_version.sh
+
+# Show git diff statistique for all repo from last tag version release
+.PHONY: repo_diff_stat_from_last_version
+repo_diff_stat_from_last_version:
+	./script/repo_diff_stat_last_version.sh
+
 # change all repo to ssh on all remote
 .PHONY: repo_use_all_ssh
 repo_use_all_ssh:

doc/DEVELOPMENT.md

@@ -66,6 +66,12 @@ Tools to display the differences between the repo and another project.
 ./script/git_change_remote.py --sync_to /path/to/project/erplibre --dry_sync
 ```
 
+## Showing repo differences with manifest develop
+To understand the divergence with the dev manifest.
+```bash
+./script/git_show_code_diff_repo_manifest.py -m ./manifest/default.dev.xml
+```
+
 ## Sync repo with another project
 Tools to synchronise the repo with another project. This will show differences and try to checkout on the same commit in all repos.
 ```bash

doc/GIT_REPO.md

@@ -82,6 +82,18 @@ git commit -am "[#ticket] subject: short sentence"
 ```
 
 ### Clean all
+Before, check changed file in all repos.
+
+```bash
+./.venv/repo forall -pc "git status -s"
+```
+
+Check the changed branch, and push changed if needed.
+
+```bash
+./script/git_show_code_diff_repo_manifest.py -m ./manifest/default.dev.xml
+```
+
 Maybe, some version diverge from your manifest. Simply clean all and relaunch your installation.
 ```bash
 ./script/clean_repo_manifest.sh

doc/RELEASE.md

@@ -6,7 +6,13 @@ Before starting, validate [manifest/default.dev.xml](../manifest/default.dev.xml
 
 ## Clean environment before generate new release
 
-Be sure all files are commit and push, this will erase everything in addons.
+Before clean, check if existing file not committed, not pushed or in stash.
+```bash
+./.venv/repo forall -pc "git stash list"
+./script/git_show_code_diff_repo_manifest.py
+```
+
+This will erase everything in addons. Useful before create docker, manifest and do a release.
 
 ```bash
 ./script/clean_repo_manifest.sh
@@ -18,6 +24,10 @@ And update all from dev to merge into prod.
 ./script/install_locally_dev.sh
 ```
 
+## Update image_db
+
+Change all default image to improve speed when restoring database. Recreate it manually. Check directory `./image_db`.
+
 ## Generate new prod and release
 
 Generate production manifest and freeze all repos versions.
@@ -49,6 +59,16 @@ git diff v#.#.#..HEAD
 ./.venv/repo forall -pc "git diff ERPLibre/v#.#.#..HEAD"
 ```
 
+Simplification tools:
+```bash
+# Show all divergence repository with production
+make repo_diff_manifest_production
+# Short version with statistique
+make repo_diff_stat_from_last_version
+# Long version
+make repo_diff_from_last_version
+```
+
 Update file [CHANGELOG.md](../CHANGELOG.md) and create a section with new version, use next command to read all changes.
 
 Create a branch release/#.#.# and create a pull request to branch master with your commit:
@@ -59,6 +79,21 @@ git commit -am "Release v#.#.#"
 
 Review by your peers, test the docker file and merge to master.
 
+## Generate image db to accelerate db installation
+
+Generate image db before tag, the image is store in directory ./image_db
+
+```bash
+make image_db_create_all
+```
+
+To test it, you need to clean caches and install it:
+
+```bash
+./script/db_restore.py --clean_cache
+./script/db_restore.py --database test --image erplibre_website
+```
+
 ## Create tag
 
 Add a tag on the commit in branch master with your release. When adding tag, be sure to update default.xml
@@ -69,7 +104,7 @@ git tag v#.#.#
 git push --tags
 # Add tags for all repo
 ./.venv/repo forall -pc "git tag ERPLibre/v#.#.#"
-./.venv/repo forall -pc "git push ERPLibre --tags"
+make tag_push_all
 ```
 
 ## Generate and push docker
@@ -77,7 +112,7 @@ git push --tags
 Important to generate container after push git tags, otherwise the git version will be wrong.
 
 When building your docker with script
-> ./script/docker_build.sh --release
+> make docker_build_release
 
 List your docker version
 > docker images

docker/README.md

@@ -114,10 +114,10 @@ And restart docker service. You can delete or move all older locations of docker
 
 # Update docker
 When building your docker with script
-> ./script/docker_build.sh
+> make docker_build
 
 List your docker version
-> docker image
+> docker images
 
 You need to push your docker image and update your tag, like 1.0.1:
 > docker push technolibre/erplibre:VERSION

poetry.lock

@@ -191,6 +191,7 @@ uvloop = ["uvloop (>=0.15.2)"]
 reference = "93c10bf9ebccf8d7cc686b0b9579f2e5e41c5328"
 type = "git"
 url = "https://github.com/psf/black.git"
+
 [[package]]
 category = "main"
 description = "Interactive plots and applications in the browser from Python"
@@ -1774,6 +1775,7 @@ whichcraft = "*"
 reference = "455ac91d2d6d651bf67ec8eae7dae516f11b1dec"
 type = "git"
 url = "https://github.com/oca/pylint-odoo.git"
+
 [[package]]
 category = "main"
 description = "Utilities and helpers for writing Pylint plugins"
@@ -2970,7 +2972,7 @@ python-versions = "*"
 version = "4.4.28"
 
 [metadata]
-content-hash = "8f717dd4d3138e704720a02bc6ca7efb301108825e0cfc0dcd16c2a8d3822b6c"
+content-hash = "8de9276db75ff5d36eae0441c17e9c88134de99323ae57b2c09d8ded790012af"
 lock-version = "1.0"
 python-versions = "^3.7.7"
 

requirements.txt

@@ -7,6 +7,7 @@ retrying
 xmltodict
 openupgradelib
 unidiff
+colorama
 
 # For OSX
 cython

run.sh

@@ -1,8 +1,19 @@
 #!/usr/bin/env bash
 source ./.venv/bin/activate
-python3 ./odoo/odoo-bin -c ./config.conf --limit-time-real 99999 --limit-time-cpu 99999 $@
+
+CONFIG_PATH="./config.conf"
+ORIGIN_CONFIG_PATH=CONFIG_PATH
+if [ ! -f "${CONFIG_PATH}" ]; then
+  CONFIG_PATH="/etc/odoo/odoo.conf"
+  if [ ! -f "${CONFIG_PATH}" ]; then
+    echo "Cannot find ERPLibre configuration ${ORIGIN_CONFIG_PATH}, did you install ERPLibre? > make install"
+    exit 1
+  fi
+fi
+
+python3 ./odoo/odoo-bin -c ${CONFIG_PATH} --limit-time-real 99999 --limit-time-cpu 99999 $@
 retVal=$?
 if [[ $retVal -ne 0 ]]; then
-    echo "Error run.sh"
+  echo "Error run.sh"
-    exit 1
+  exit 1
 fi

script/code_generator/install_and_test_code_generator.sh

@@ -12,8 +12,14 @@ fi
 ./script/repo_revert_git_diff_date_from_code_generator.py
 # Remove pot and po diff
 cd $3
-BRANCH=$(git branch --show-current)
+# git 2.22 and more, else use next command
-git restore --source="${BRANCH}" "*.po*"
+#BRANCH=$(git branch --show-current)
+BRANCH=$(git rev-parse --abbrev-ref HEAD)
+
+# Support old version git < 2.23.0
+# git restore --source="${BRANCH}" "*.po*"
+git checkout -- "*.po*"
+
 cd -
 ./script/maintenance/black.sh $3
 echo "TEST ${2}"

script/csv/compare_database_application.py

@@ -0,0 +1,67 @@
+#!./.venv/bin/python
+import os
+import sys
+import argparse
+import logging
+import csv
+
+new_path = os.path.normpath(os.path.join(os.path.dirname(__file__), ".."))
+sys.path.append(new_path)
+
+logging.basicConfig(level=os.environ.get("LOGLEVEL", "INFO"))
+
+_logger = logging.getLogger(__name__)
+
+
+def get_config():
+    """Parse command line arguments, extracting the config file name,
+    returning the union of config file and command line arguments
+
+    :return: dict of config file settings and command line arguments
+    """
+    # TODO update description
+    parser = argparse.ArgumentParser(
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        description="""\
+        Take two csv and show difference, intersection and complementarity
+""",
+        epilog="""\
+""",
+    )
+    parser.add_argument("--csv_1", required=True, help="CSV file first")
+    parser.add_argument("--csv_2", required=True, help="CSV file second")
+    parser.add_argument("--compare_key", required=True, help="Key to search and compare with it.")
+    args = parser.parse_args()
+    return args
+
+
+def main():
+    config = get_config()
+
+    with open(config.csv_1, mode='r') as csv_file_1:
+        with open(config.csv_2, mode='r') as csv_file_2:
+            csv_reader_1 = csv.DictReader(csv_file_1)
+            csv_reader_2 = csv.DictReader(csv_file_2)
+            lst_csv_1 = [a.get(config.compare_key) for a in csv_reader_1]
+            lst_csv_2 = [a.get(config.compare_key) for a in csv_reader_2]
+    set_csv_1 = set(lst_csv_1)
+    set_csv_2 = set(lst_csv_2)
+
+    total = set_csv_1.union(set_csv_2)
+    same = set_csv_1.intersection(set_csv_2)
+    difference_1 = set_csv_1.difference(set_csv_2)
+    difference_2 = set_csv_2.difference(set_csv_1)
+    print(f"{len(total)} total")
+    print(f"{len(same)} same")
+    if same:
+        print(same)
+    print(f"{len(difference_1)} difference csv 1 to csv 2")
+    if difference_1:
+        print(difference_1)
+    print(f"{len(difference_2)} difference csv 2 to csv 1")
+    if difference_2:
+        print(difference_2)
+
+
+if __name__ == "__main__":
+    main()

script/db_restore.py

@@ -3,6 +3,8 @@ import os
 import sys
 import argparse
 import logging
+import configparser
+import getpass
 from subprocess import check_output
 
 new_path = os.path.normpath(os.path.join(os.path.dirname(__file__), ".."))
@@ -49,34 +51,67 @@ def get_config():
     return args
 
 
-def main():
+def get_master_password():
-    config = get_config()
+    try:
+        # _logger.info("You have 5 secondes to add master password...")
+        pa = getpass.getpass(prompt="\nEnter master password... ")
+        return pa
+    except getpass.GetPassWarning:
+        _logger.error("Password echoed, danger!")
 
-    # Get list of database
+
-    arg = "./.venv/bin/python3 ./odoo/odoo-bin db --list"
+def get_list_db_cache(arg_base):
+    arg = f"{arg_base} --list"
     out = check_output(arg.split(" ")).decode()
     lst_db = out.strip().split("\n")
     lst_db_cache = [a for a in lst_db if a.startswith("_cache_")]
+    return lst_db, lst_db_cache
+
+
+def main():
+    config = get_config()
+
+    arg_base = "./.venv/bin/python3 ./odoo/odoo-bin db"
+
+    # check if need master password from config file
+    has_config_file = True
+    config_path = "./config.py"
+    if not os.path.isfile(config_path):
+        config_path = "/etc/odoo/odoo.conf"
+        if not os.path.isfile(config_path):
+            has_config_file = False
+    if has_config_file:
+        config_parser = configparser.ConfigParser()
+        config_parser.read(config_path)
+
+        has_admin_password = config_parser.get("options", "admin_passwd")
+        if has_admin_password:
+            master_password = get_master_password()
+            if not master_password:
+                _logger.error("Missing master password, cancel transaction.")
+                sys.exit(1)
+            else:
+                arg_base += f" --master_password={master_password}"
+        else:
+            _logger.info("No master password needed... Continue")
+
+    # Get list of database
+    lst_db, lst_db_cache = get_list_db_cache(arg_base)
 
     if config.clean_cache:
         for db in lst_db_cache:
             _logger.info(f"## Delete {db} ##")
-            arg = (
+            arg = f"{arg_base} --drop --database {db}"
-                "./.venv/bin/python3 ./odoo/odoo-bin db --drop --database"
-                f" {db}"
-            )
             out = check_output(arg.split(" ")).decode()
             print(out)
+        lst_db, lst_db_cache = get_list_db_cache(arg_base)
 
     if config.database:
         cache_database = f"_cache_{config.image}"
         # Drop db
         if config.database in lst_db:
             _logger.info(f"## Drop {config.database} ##")
-            arg = (
+            arg = f"{arg_base} --drop --database {config.database}"
-                "./.venv/bin/python3 ./odoo/odoo-bin db --drop --database"
-                f" {config.database}"
-            )
             out = check_output(arg.split(" ")).decode()
             print(out)
         # Check cache exist
@@ -86,7 +121,7 @@ def main():
                 f" {config.image} ##"
             )
             arg = (
-                "./.venv/bin/python3 ./odoo/odoo-bin db --restore"
+                f"{arg_base} --restore"
                 f" --restore_image {config.image} --database {cache_database}"
             )
             out = check_output(arg.split(" ")).decode()
@@ -96,7 +131,7 @@ def main():
             f"## Clone cache {cache_database} to database {config.database} ##"
         )
         arg = (
-            "./.venv/bin/python3 ./odoo/odoo-bin db --clone --from_database"
+            f"{arg_base} --clone --from_database"
             f" {cache_database} --database {config.database}"
         )
         out = check_output(arg.split(" ")).decode()

script/docker/docker_exec.sh

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+CURRENT=$(pwd)
+BASENAME=$(basename "${CURRENT}")
+
+docker exec -u root -ti ${BASENAME}_ERPLibre_1 bash

script/docker/docker_gen_config.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+
+CURRENT=$(pwd)
+BASENAME=$(basename "${CURRENT}")
+
+docker exec -u root -ti ${BASENAME}_ERPLibre_1 /bin/bash -c "\
+cd /ERPLibre; \
+./docker/repo_manifest_gen_org_prefix_path.py /ERPLibre/addons /etc/odoo/odoo.conf /etc/odoo/odoo.conf; \
+"

script/docker/docker_make_test.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+
+CURRENT=$(pwd)
+BASENAME=$(basename "${CURRENT}")
+
+docker exec -u root -ti ${BASENAME}_ERPLibre_1 /bin/bash -c "\
+cd /ERPLibre; \
+time make test; \
+"

script/docker/docker_repo_show_status.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+CURRENT=$(pwd)
+BASENAME=$(basename "${CURRENT}")
+
+docker exec -u root -ti ${BASENAME}_ERPLibre_1 /bin/bash -c "\
+cd /ERPLibre; \
+./.venv/repo forall -pc 'git status -s'; \
+echo ''; \
+git status -s; \
+"

script/git_show_code_diff_repo_manifest.py

@@ -4,7 +4,10 @@ import sys
 import argparse
 import logging
 from git import Repo
-from git.exc import GitCommandError
+from git.exc import GitCommandError, NoSuchPathError
+from colorama import Fore
+from colorama import Style
+from collections import defaultdict
 
 new_path = os.path.normpath(os.path.join(os.path.dirname(__file__), ".."))
 sys.path.append(new_path)
@@ -30,7 +33,7 @@ def get_config():
     parser.add_argument(
         "-m",
         "--manifest",
-        required=True,
+        default="./default.xml",
         help="The manifest to compare with actual code.",
     )
     args = parser.parse_args()
@@ -47,6 +50,7 @@ def main():
     default_branch_name = default_remote.get(
         "@revision", git_tool.default_branch
     )
+    dct_result = defaultdict(int)
     i = 0
     total = len(dct_project)
     for name, project in dct_project.items():
@@ -54,12 +58,18 @@ def main():
         path = project.get("@path")
         print(f"{i}/{total} - {path}")
         branch_name = project.get("@revision", default_branch_name)
-        organization = project.get("@remote")
+        organization = project.get("@remote", git_tool.default_project_name)
-        if not organization:
+
-            print(f"ERROR missing @remote on project {path}.")
+        try:
+            git_repo = Repo(path)
+        except NoSuchPathError:
+            print(
+                f"{Fore.YELLOW}Warning{Style.RESET_ALL} missing project"
+                f" {path}."
+            )
+            dct_result["WARNING"] += 1
             continue
 
-        git_repo = Repo(path)
         value = git_repo.git.branch("--show-current")
         if not value:
             # TODO maybe need to check divergence with local branch and not remote branch
@@ -69,19 +79,79 @@ def main():
                     f"{organization}/{branch_name}"
                 )
             except GitCommandError:
-                print("ERROR Something wrong with this repo.")
+                # Cannot get information
+                if branch_name == commit_head:
+                    print(
+                        f"{Fore.GREEN}PASS{Style.RESET_ALL} Not on specified"
+                        " branch, no divergence"
+                    )
+                    dct_result["PASS"] += 1
+                else:
+                    print(
+                        f"{Fore.RED}ERROR{Style.RESET_ALL} manifest revision"
+                        f" is {branch_name} and commit {commit_head}."
+                    )
+                    dct_result["ERROR"] += 1
                 continue
             if commit_branch != commit_head:
-                print("WARNING Not on specified branch, got a divergence.")
+                print(
+                    f"{Fore.YELLOW}WARNING{Style.RESET_ALL} Not on specified"
+                    " branch, got a divergence."
+                )
+                dct_result["WARNING"] += 1
             else:
-                print("PASS Not on specified branch, no divergence.")
+                print(
+                    f"{Fore.GREEN}PASS{Style.RESET_ALL} Not on specified"
+                    " branch, no divergence"
+                )
+                dct_result["PASS"] += 1
         elif branch_name != value:
-            print(
+            value_hash = git_repo.git.rev_parse(value)
-                f"ERROR, manifest revision is {branch_name} and actual"
+            if git_repo.git.rev_parse(branch_name) == value_hash:
-                f" revision is {value}."
+                print(
-            )
+                    f"{Fore.GREEN}PASS{Style.RESET_ALL} Not same branch, no"
+                    " divergence"
+                )
+                dct_result["PASS"] += 1
+            else:
+                # Check if the new branch is pushed
+                commit_branch = git_repo.git.rev_parse(
+                    f"{organization}/{value}"
+                )
+                if commit_branch == value_hash:
+                    print(
+                        f"{Fore.YELLOW}WARNING{Style.RESET_ALL} New branch"
+                        f" '{value}', divergence, but it's push on remote."
+                    )
+                    dct_result["WARNING"] += 1
+                else:
+                    print(
+                        f"{Fore.RED}ERROR{Style.RESET_ALL} manifest revision"
+                        f" is {branch_name} and actual revision is {value}."
+                    )
+                    dct_result["ERROR"] += 1
         else:
-            print("PASS")
+            print(f"{Fore.GREEN}PASS{Style.RESET_ALL}")
+            dct_result["PASS"] += 1
+
+    str_result = ""
+    if dct_result["PASS"]:
+        str_result += (
+            f"{Fore.GREEN}PASS: {dct_result['PASS']}{Style.RESET_ALL}"
+        )
+    if dct_result["WARNING"]:
+        if str_result:
+            str_result += " "
+        str_result += (
+            f"{Fore.YELLOW}WARNING: {dct_result['WARNING']}{Style.RESET_ALL}"
+        )
+    if dct_result["ERROR"]:
+        if str_result:
+            str_result += " "
+        str_result += (
+            f"{Fore.RED}ERROR: {dct_result['ERROR']}{Style.RESET_ALL}"
+        )
+    print(str_result)
 
 
 if __name__ == "__main__":

script/manifest/compare_backup.py

@@ -0,0 +1,99 @@
+#!./.venv/bin/python
+import os
+import sys
+import argparse
+import logging
+import zipfile
+import json
+from colorama import Fore
+from colorama import Style
+
+new_path = os.path.normpath(os.path.join(os.path.dirname(__file__), ".."))
+sys.path.append(new_path)
+
+logging.basicConfig(level=os.environ.get("LOGLEVEL", "INFO"))
+
+_logger = logging.getLogger(__name__)
+
+
+def get_config():
+    """Parse command line arguments, extracting the config file name,
+    returning the union of config file and command line arguments
+
+    :return: dict of config file settings and command line arguments
+    """
+    # TODO update description
+    parser = argparse.ArgumentParser(
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        description="""\
+        Take two backup and show difference, intersection and complementarity of modules in manifest.json
+""",
+        epilog="""\
+""",
+    )
+    parser.add_argument("--backup_file_1", help="Backup file path first")
+    parser.add_argument("--backup_file_2", help="Backup file path second")
+    parser.add_argument("--backup_1", help="Backup name first")
+    parser.add_argument("--backup_2", help="Backup name second")
+    args = parser.parse_args()
+
+    die(bool(args.backup_file_1) and bool(args.backup_1),
+        "Take only --backup_file_1 or --backup_1")
+    die(not bool(args.backup_file_1) and not bool(args.backup_1),
+        "Missing --backup_file_1 or --backup_1")
+    die(bool(args.backup_file_2) and bool(args.backup_2),
+        "Take only --backup_file_2 or --backup_2")
+    die(not bool(args.backup_file_2) and not bool(args.backup_2),
+        "Missing --backup_file_2 or --backup_2")
+
+    return args
+
+
+def main():
+    config = get_config()
+
+    if config.backup_file_1:
+        file_path_1 = config.backup_file_1
+    else:
+        file_path_1 = os.path.join("image_db", f"{config.backup_1}.zip")
+
+    if config.backup_file_2:
+        file_path_2 = config.backup_file_2
+    else:
+        file_path_2 = os.path.join("image_db", f"{config.backup_2}.zip")
+        
+    with zipfile.ZipFile(file_path_1, 'r') as zip_ref:
+        manifest_file_1 = zip_ref.open("manifest.json")
+
+    with zipfile.ZipFile(file_path_2, 'r') as zip_ref:
+        manifest_file_2 = zip_ref.open("manifest.json")
+
+    json_manifest_file_1 = json.load(manifest_file_1)
+    json_manifest_file_2 = json.load(manifest_file_2)
+    set_1 = set(json_manifest_file_1.get("modules").keys())
+    set_2 = set(json_manifest_file_2.get("modules").keys())
+
+    total = set_1.union(set_2)
+    same = set_1.intersection(set_2)
+    difference_1 = set_1.difference(set_2)
+    difference_2 = set_2.difference(set_1)
+    print(f"{len(total)} total")
+    print(f"{len(same)} same")
+    if same:
+        print(same)
+    print(f"{Fore.BLUE}{len(difference_1)}{Style.RESET_ALL} difference manifest 1 to manifest 2")
+    if difference_1:
+        print(difference_1)
+    print(f"{Fore.MAGENTA}{len(difference_2)}{Style.RESET_ALL} difference manifest 2 to manifest 1")
+    if difference_2:
+        print(difference_2)
+
+
+def die(cond, message, code=1):
+    if cond:
+        print(message, file=sys.stderr)
+        sys.exit(code)
+
+
+if __name__ == "__main__":
+    main()

script/repo_diff_last_version.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+LAST_TAG=$(git describe --tags $(git rev-list --tags --max-count=1))
+./.venv/repo forall -pc "git diff ERPLibre/${LAST_TAG}..HEAD"
+
+# For actual repo
+git diff ${LAST_TAG}..HEAD

script/repo_diff_stat_last_version.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+
+Color_Off='\033[0m' # Text Reset
+BOLD='\033[1m'      # Black
+
+LAST_TAG=$(git describe --tags $(git rev-list --tags --max-count=1))
+./.venv/repo forall -pc "git diff --stat ERPLibre/${LAST_TAG}..HEAD"
+
+echo ""
+echo -e "${BOLD}project /${Color_Off}"
+
+# For actual repo
+git diff --stat ${LAST_TAG}..HEAD

script/tag_push_all.py

@@ -0,0 +1,70 @@
+#!./.venv/bin/python
+import os
+import sys
+import argparse
+import logging
+from git import Repo  # pip install gitpython
+from retrying import retry  # pip install retrying
+from colorama import Fore
+from colorama import Style
+
+new_path = os.path.normpath(os.path.join(os.path.dirname(__file__), ".."))
+sys.path.append(new_path)
+
+from script.git_tool import GitTool
+
+_logger = logging.getLogger(__name__)
+
+
+def get_config():
+    """Parse command line arguments, extracting the config file name,
+    returning the union of config file and command line arguments
+
+    :return: dict of config file settings and command line arguments
+    """
+    # TODO update description
+    parser = argparse.ArgumentParser(
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        description="""Push all addons git.""",
+        epilog="""\
+""",
+    )
+    parser.add_argument(
+        "-m",
+        "--manifest",
+        default="./default.xml",
+        help="The manifest to compare with actual code.",
+    )
+    args = parser.parse_args()
+    return args
+
+
+def main():
+    config = get_config()
+    git_tool = GitTool()
+
+    dct_remote, dct_project, default_remote = git_tool.get_manifest_xml_info(
+        filename=config.manifest, add_root=True
+    )
+    i = 0
+    total = len(dct_project)
+    for name, project in dct_project.items():
+        i += 1
+        path = project.get("@path")
+        print(f"{i}/{total} - {path}")
+        organization = project.get("@remote", git_tool.default_project_name)
+
+        try:
+            git_repo = Repo(path)
+            retry(wait_exponential_multiplier=1000, stop_max_delay=15000)(
+                git_repo.git.push
+            )(organization, "--tags")
+        except:
+            print(
+                f"{Fore.RED}ERROR{Style.RESET_ALL} cannot push --tags for path"
+                f" {path} organization {organization}"
+            )
+
+
+if __name__ == "__main__":
+    main()