diff --git a/.gitignore b/.gitignore index 5ea8a44..8f8d39f 100644 --- a/.gitignore +++ b/.gitignore @@ -24,5 +24,5 @@ get-poetry.py artifacts wkhtmltox* cache - node_modules +docker-compose.override.yml diff --git a/README.md b/README.md index d4e5f2e..c6246a0 100644 --- a/README.md +++ b/README.md @@ -1,33 +1,53 @@ # ERPLibre documentation + Select a guide to install your environment. ## Easy way to run locally + Into Ubuntu, minimal dependency: + ```bash -sudo apt install make git +sudo apt install make git curl ``` + +Into Ubuntu, developper dependency: + +```bash +sudo apt install make build-essential libssl-dev zlib1g-dev libreadline-dev libsqlite3-dev curl llvm libncurses5-dev libncursesw5-dev xz-utils tk-dev liblzma-dev libbz2-dev libldap2-dev libsasl2-dev +``` + Clone the project: + ```bash git clone https://github.com/ERPLibre/ERPLibre.git cd ERPLibre ``` + Support Ubuntu 18.04, 20.04 and OSX. The installation duration is more than 30 minutes. + ```bash make install ``` + Update your configuration if you need to run from another interface than 127.0.0.1, file `config.conf` + ``` xmlrpc_interface = 0.0.0.0 ``` + Ready to execute: + ```bash make run ``` ## Easy way to run docker -First, install dependencies to run docker, check script `./script/install/install_ubuntu_docker.sh`. You need docker and docker-compose. + +First, install dependencies to run docker, check script `./script/install/install_ubuntu_docker.sh`. You need docker and +docker-compose. The docker volume is bound to the directory name, therefore create a unique directory name and run: + ```bash wget https://raw.githubusercontent.com/ERPLibre/ERPLibre/v1.5.0/docker-compose.yml docker compose up -d @@ -36,22 +56,29 @@ docker compose up -d For more information, read [Docker guide](./docker/README.md). ## Discover guide + [Guide to run ERPLibre in discover to learn it](./doc/DISCOVER.md). ## Production guide + [Guide to run ERPLibre in production server](./doc/PRODUCTION.md). ## Development guide + [Guide to run ERPLibre in development environment](./doc/DEVELOPMENT.md). # Execution + [Guide to run ERPLibre with different case](./doc/RUN.md). # git-repo + To change repository like addons, see [GIT_REPO.md](doc/GIT_REPO.md) # Test + Execute ERPLibre test with his code generator. + ```bash time make test_full_fast ``` diff --git a/doc/DEVELOPMENT.md b/doc/DEVELOPMENT.md index 381c614..9dde2d3 100644 --- a/doc/DEVELOPMENT.md +++ b/doc/DEVELOPMENT.md @@ -61,7 +61,8 @@ Execute to generate Repo manifest ## Move database from prod to dev -Copy the database image into `./image_db/prod_client.zip` and run `make db_restore_prod_client`. This will create a database +Copy the database image into `./image_db/prod_client.zip` and run `make db_restore_prod_client`. This will create a +database named `prod_client` ready to test. When moving database from prod to your dev environment, you want to remove email servers, backups and install user test @@ -220,6 +221,24 @@ Update poetry, `./script/poetry/poetry_update.py`. Create docker, `make docker_build`. +### Python version major change + +When you need to change python 3.7.17 to 3.8.10, do : + +```bash +rm -r .venv +make install_dev +./.venv/bin/poetry lock --no-update +``` + +And update file script/install/install_locally.sh + +`ln -fs ${EL_HOME_ODOO}/odoo ${EL_HOME}/.venv/lib/python3.7/site-packages/` + +to + +`ln -fs ${EL_HOME_ODOO}/odoo ${EL_HOME}/.venv/lib/python3.8/site-packages/` + # Pull request ## Show all pull requests from organization diff --git a/doc/FAQ.md b/doc/FAQ.md index d1811a2..a437b2f 100644 --- a/doc/FAQ.md +++ b/doc/FAQ.md @@ -87,10 +87,14 @@ git cherry-pick -m 1 --strategy-option theirs HASH ## git update manifest -### Error fatal: unable to allocate any listen sockets on port 9418 +### Service git-daemon already running, error bind or Error fatal: unable to allocate any listen sockets on port 9418 + +This error occur when force stop (ctrl+c) a script like `./script/manifest/update_manifest_local_dev.sh` + +The error into console is similar to `Could not bind to 0.0.0.0: Address already in use` ```bash -pkill git-daemon +pkill -f git-daemon ``` ## git-repo @@ -149,21 +153,28 @@ Add line at the end of config.conf limit_memory_hard = 0 ``` -### Docker - All interface bind docker +## Docker - All interface bind docker -Create a subnet +### Error non-overlapping IPv4 address pool + +You got this error when you start a +docker-compose: `ERROR: could not find an available, non-overlapping IPv4 address pool among the defaults to assign to the network` + +It's because the subnet is limited, you need to change it. + +Create a subnet : ```bash -sudo docker network create localnetwork --subnet 10.0.1.0/24 +docker network create localnetwork --subnet 10.0.1.0/24 ``` -And create `docker-compose.override.yml` at root project with - +Create a new file `docker-compose.override.yml` at the root of ERPLibre, at same level of your docker-compose.yml and +fill with: ```yaml version: '3' networks: - default: - external: - name: localnetwork + default: + external: + name: localnetwork ``` diff --git a/doc/POETRY.md b/doc/POETRY.md index 543a7f9..8c3adc6 100644 --- a/doc/POETRY.md +++ b/doc/POETRY.md @@ -34,3 +34,13 @@ Change version in file `./script/install/install_locally.sh` into constant `POET Erase directory `~/.poetry` and `./get-poetry.py`. Run installation script for OS, check `./script/install/install_locally.sh`. + +# FAQ + +## Got error "file could not be opened successfully" + +Delete cache at root of the project + +```bash +rm -r cache +``` diff --git a/docker/Dockerfile.base b/docker/Dockerfile.base index cb8eb25..244daff 100644 --- a/docker/Dockerfile.base +++ b/docker/Dockerfile.base @@ -31,7 +31,11 @@ RUN apt-get update \ vim \ htop \ tig \ + cmake \ wget \ + tig \ + vim \ + htop \ make \ libssl-dev \ zlib1g-dev \ @@ -61,6 +65,7 @@ RUN apt-get update \ libncursesw5-dev \ libffi-dev \ uuid-dev \ + swig \ && curl -o wkhtmltox.deb -sSL https://github.com/wkhtmltopdf/packaging/releases/download/0.12.6-1/wkhtmltox_0.12.6-1.buster_amd64.deb \ && echo 'd9f259a67e05e1c221d48b504453645e6c491fab wkhtmltox.deb' | sha1sum -c - \ && apt-get install -y --no-install-recommends ./wkhtmltox.deb \ diff --git a/script/git/git_repo_manifest.py b/script/git/git_repo_manifest.py index 536eb4a..180ddbf 100755 --- a/script/git/git_repo_manifest.py +++ b/script/git/git_repo_manifest.py @@ -40,12 +40,22 @@ def get_config(): action="store_true", help="Create a new manifest and clear old configuration.", ) + parser.add_argument( + "--keep_origin", + action="store_true", + help="Create origin remote. TODO.", + ) parser.add_argument( "-m", "--manifest", default="manifest/default.dev.xml", help="The manifest file path to generate.", ) + parser.add_argument( + "--default_branch", + default=False, + help="The manifest default branch.", + ) args = parser.parse_args() return args @@ -78,12 +88,16 @@ def main(): else: dct_remote = {} dct_project = {} + kwargs = {} + if config.default_branch: + kwargs["default_branch"] = config.default_branch git_tool.generate_repo_manifest( lst_repo_organization, output=f"{config.dir}{config.manifest}", dct_remote=dct_remote, dct_project=dct_project, - keep_original=True, + keep_original=config.keep_origin, + **kwargs, ) git_tool.generate_generate_config() diff --git a/script/git/git_tool.py b/script/git/git_tool.py index 8747ec4..1281805 100644 --- a/script/git/git_tool.py +++ b/script/git/git_tool.py @@ -445,6 +445,7 @@ class GitTool: dct_project={}, default_remote=None, keep_original=False, + default_branch=DEFAULT_BRANCH, ): """ Generate repo manifest @@ -455,6 +456,7 @@ class GitTool: :param default_remote: dict of default remote :param keep_original: if True, can manage multiple organization with same name, but with different fetch url + :param default_branch: default branch name :return: """ if not output: @@ -469,15 +471,17 @@ class GitTool: # Fill with configuration for dct_value in dct_remote.values(): - lst_remote.append( - OrderedDict( - [ - ("@name", dct_value.get("@name")), - ("@fetch", dct_value.get("@fetch")), - ] + remote_name = dct_value.get("@name") + if remote_name not in lst_remote_name: + lst_remote.append( + OrderedDict( + [ + ("@name", remote_name), + ("@fetch", dct_value.get("@fetch")), + ] + ) ) - ) - lst_remote_name.append(dct_value.get("@name")) + lst_remote_name.append(remote_name) for dct_value in dct_project.values(): lst_project_info = [ ("@name", dct_value.get("@name")), @@ -519,7 +523,7 @@ class GitTool: OrderedDict( [ ("@remote", repo.original_organization), - ("@revision", DEFAULT_BRANCH), + ("@revision", default_branch), ("@sync-j", "4"), ("@sync-c", "true"), ] @@ -572,7 +576,7 @@ class GitTool: OrderedDict( [ ("@remote", default_remote.get("@remote")), - ("@revision", DEFAULT_BRANCH), + ("@revision", default_branch), ("@sync-j", "4"), ("@sync-c", "true"), ] diff --git a/script/git/repo_remove_auto_install.py b/script/git/repo_remove_auto_install.py index 84aa149..dac6eda 100755 --- a/script/git/repo_remove_auto_install.py +++ b/script/git/repo_remove_auto_install.py @@ -13,7 +13,7 @@ new_path = os.path.normpath( ) sys.path.append(new_path) -from script.git.git_tool import GitTool +from script.git import git_tool as g_tool _logger = logging.getLogger(__name__) @@ -39,34 +39,65 @@ def get_config(): default="./", help="Path of repo to change remote, including submodule.", ) + parser.add_argument( + "--addons_dir", + help=( + "Path of addons to remove auto_install. If empty, will take all" + " repo from manifest." + ), + ) + parser.add_argument( + "--ignore_commit", + action="store_true", + help="Will not do git commit and push.", + ) + parser.add_argument( + "--ignore_push", + action="store_true", + help="Will not do git push.", + ) args = parser.parse_args() return args def main(): config = get_config() - git_tool = GitTool() + git_tool = g_tool.GitTool() - lst_repo = git_tool.get_source_repo_addons( - repo_path=config.dir, add_repo_root=False - ) - lst_repo_organization = [ - git_tool.get_transformed_repo_info_from_url( - a.get("url"), - repo_path=config.dir, - get_obj=True, - is_submodule=a.get("is_submodule"), - sub_path=a.get("sub_path"), - revision=a.get("revision"), - clone_depth=a.get("clone_depth"), + if not config.addons_dir: + lst_repo = git_tool.get_source_repo_addons( + repo_path=config.dir, add_repo_root=False ) - for a in lst_repo - ] + lst_repo_organization = [ + git_tool.get_transformed_repo_info_from_url( + a.get("url"), + repo_path=config.dir, + get_obj=True, + is_submodule=a.get("is_submodule"), + sub_path=a.get("sub_path"), + revision=a.get("revision"), + clone_depth=a.get("clone_depth"), + ) + for a in lst_repo + ] + else: + organization = os.path.basename(config.addons_dir) + # TODO information is wrong, need to extract organisation and repo_name + lst_repo_organization = [ + g_tool.Struct( + **{ + "organization": organization, + "path": config.addons_dir, + "relative_path": config.addons_dir, + "repo_name": config.addons_dir, + } + ) + ] lst_ignore_repo = ["odoo"] i = 0 - total = len(lst_repo) + total = len(lst_repo_organization) branch_name = "12.0_dev" for repo in lst_repo_organization: i += 1 @@ -77,29 +108,31 @@ def main(): print(f"Ignore {repo.repo_name}.") continue - git_repo = Repo(repo.relative_path) - # Force checkout branch if exist - try: - git_repo.git.checkout(branch_name) - is_checkout_branch = True - except GitCommandError: + if not config.ignore_commit: + git_repo = Repo(repo.relative_path) + # Force checkout branch if exist try: - git_repo.git.checkout( - "-t", f"{repo.organization}/{branch_name}" - ) + git_repo.git.checkout(branch_name) is_checkout_branch = True except GitCommandError: - pass + try: + git_repo.git.checkout( + "-t", f"{repo.organization}/{branch_name}" + ) + is_checkout_branch = True + except GitCommandError: + pass has_change = get_manifest_external_dependencies(repo) - if has_change: + if has_change and not config.ignore_commit: if not is_checkout_branch: git_repo.git.checkout("-b", branch_name) # change branch, commit and push git_repo.git.add(".") git_repo.git.commit("-m", "Set all module auto_install at False") - git_repo.git.push("-u", repo.organization, branch_name) + if not config.ignore_push: + git_repo.git.push("-u", repo.organization, branch_name) def get_lst_manifest_py(relative_path): diff --git a/script/ide/pycharm_configuration.py b/script/ide/pycharm_configuration.py index d5d22bb..4df53c2 100755 --- a/script/ide/pycharm_configuration.py +++ b/script/ide/pycharm_configuration.py @@ -5,6 +5,7 @@ import logging import os import subprocess import sys +import glob import xmltodict @@ -16,7 +17,6 @@ PROJECT_NAME = os.path.basename(os.getcwd()) IDEA_PATH = "./.idea" IDEA_MISC = os.path.join(IDEA_PATH, "misc.xml") IDEA_WORKSPACE = os.path.join(IDEA_PATH, "workspace.xml") -IDEA_PROJECT_IML = os.path.join(IDEA_PATH, PROJECT_NAME + ".iml") PATH_EXCLUDE_FOLDER = "./conf/pycharm_exclude_folder.txt" if os.path.isfile(PATH_EXCLUDE_FOLDER): @@ -99,16 +99,20 @@ def main(): f"Missing {IDEA_PATH} path, are you sure you run this script at root" " of the project, are you sure you have a Pycharm project?", ) + # Find a file ended by iml and take it + project_path = None + for file in glob.glob(IDEA_PATH + "/*.iml"): + project_path = file + break die( - not os.path.isfile(IDEA_PROJECT_IML), - f"Missing {IDEA_PROJECT_IML} file, wait after Pycharm analyze file to" + not project_path, + f"Missing .iml file into {IDEA_PATH}, wait after Pycharm analyze file to" " create Python environnement.", ) + if config.init: has_execute = True - read_xml_and_execute( - IDEA_PROJECT_IML, add_exclude_folder, "init", config - ) + read_xml_and_execute(project_path, add_exclude_folder, "init", config) die( not os.path.isfile(IDEA_MISC), diff --git a/script/install/install_OSX_dependency.sh b/script/install/install_OSX_dependency.sh index 1728a08..e6e2990 100755 --- a/script/install/install_OSX_dependency.sh +++ b/script/install/install_OSX_dependency.sh @@ -31,6 +31,7 @@ brew link wget echo "\n--- Installing extra --" brew install parallel +brew install swig echo "\n---- Installing nodeJS NPM and rtlcss for LTR support ----" brew install nodejs npm openssl sudo npm install -g rtlcss diff --git a/script/install/install_debian_dependency.sh b/script/install/install_debian_dependency.sh index 5afba64..5bf2149 100755 --- a/script/install/install_debian_dependency.sh +++ b/script/install/install_debian_dependency.sh @@ -63,7 +63,7 @@ sudo su - postgres -c "createuser -s ${EL_USER}" 2>/dev/null || true # Install Dependencies #-------------------------------------------------- echo -e "\n--- Installing debian dependency --" -sudo apt-get install git build-essential wget libxslt-dev libzip-dev libldap2-dev libsasl2-dev gdebi-core libffi-dev libbz2-dev parallel pysassc -y +sudo apt-get install git build-essential wget libxslt-dev libzip-dev libldap2-dev libsasl2-dev gdebi-core libffi-dev libbz2-dev parallel pysassc swig -y retVal=$? if [[ $retVal -ne 0 ]]; then echo "apt-get debian tool installation error." diff --git a/script/poetry/poetry_update.py b/script/poetry/poetry_update.py index 04e8cbc..3bcb345 100755 --- a/script/poetry/poetry_update.py +++ b/script/poetry/poetry_update.py @@ -89,8 +89,8 @@ def combine_requirements(config): ignore_requirements = ["sys_platform == 'win32'", "python_version < '3.7'"] dct_requirements = defaultdict(set) lst_requirements_file = get_lst_requirements_txt() - lst_requirements_with_condition = set() - dct_special_condition = defaultdict(list) + # lst_requirements_with_condition = set() + # dct_special_condition = defaultdict(list) dct_requirements_module_filename = defaultdict(list) for requirements_filename in lst_requirements_file: with open(requirements_filename, "r") as f: @@ -98,6 +98,19 @@ def combine_requirements(config): b = a.strip() if not b or b[0] == "#": continue + if " @ " in b: + # Support when requirement line is like "package @ git+https://URL" + b = b.split("@ ")[1] + if "#" in b: + # remove comments at the end of module + b = b[: b.index("#")].strip() + comment_depend = "" + if except_sign in b: + # TODO support python_version into comment_depend, check odoo/requirements.txt + b, comment_depend = b.split(except_sign) + b = b.strip() + comment_depend = comment_depend.strip() + # print(comment_depend) # Regroup requirement for sign in lst_sign: @@ -113,17 +126,18 @@ def combine_requirements(config): module_name = b[: b.find(sign)] module_name = module_name.strip() # Special condition for ";", ignore it - if except_sign in b: - for ignore_string in ignore_requirements: - if ignore_string in b: - break - if ignore_string in b: - break - lst_requirements_with_condition.add(module_name) - value = b[: b.find(except_sign)].strip() - dct_special_condition[module_name].append(b) - else: - value = b + # if except_sign in b: + # for ignore_string in ignore_requirements: + # if ignore_string in b: + # break + # if ignore_string in b: + # break + # # lst_requirements_with_condition.add(module_name) + # value = b[: b.find(except_sign)].strip() + # # dct_special_condition[module_name].append(b) + # else: + # value = b + value = b dct_requirements[module_name].add(value) filename = str(requirements_filename) dct_requirements_module_filename[value].append( @@ -136,7 +150,9 @@ def combine_requirements(config): dct_requirements_module_filename[b].append(filename) lst_requirements.append(b) - dct_requirements = get_manifest_external_dependencies(dct_requirements) + dct_requirements = get_manifest_external_dependencies( + dct_requirements, lst_sign + ) # Merge all requirement by insensitive dct_requirement_insensitive = {} @@ -285,12 +301,13 @@ def combine_requirements(config): lst_ignored_key = [] for key in dct_requirements.keys(): for ignored in lst_ignore: - if ignored == key: + if ignored == key.strip(): lst_ignored_key.append(key) for key in lst_ignored_key: del dct_requirements[key] with open("./.venv/build_dependency.txt", "w") as f: + # TODO remove all comment f.writelines([f"{list(a)[0]}\n" for a in dct_requirements.values()]) @@ -340,7 +357,7 @@ def delete_dependency_poetry(pyproject_filename): toml.dump(dct_pyproject, f) -def get_manifest_external_dependencies(dct_requirements): +def get_manifest_external_dependencies(dct_requirements, lst_sign): lst_manifest_file = get_lst_manifest_py() lst_dct_ext_depend = [] for manifest_file in lst_manifest_file: @@ -358,11 +375,16 @@ def get_manifest_external_dependencies(dct_requirements): if not python: continue for depend in python: - requirement = dct_requirements.get(depend) + # TODO duplicate code, check combine_requirements() + module_name = depend + for sign in lst_sign: + if sign in depend: + module_name = depend[: depend.find(sign)].strip() + requirement = dct_requirements.get(module_name) if requirement: requirement.add(depend) else: - dct_requirements[depend] = set([depend]) + dct_requirements[module_name] = set([depend]) return dct_requirements