diff --git a/private/.gitignore b/private/.gitignore index 5e440ed..12cb007 100644 --- a/private/.gitignore +++ b/private/.gitignore @@ -1 +1,6 @@ *.kdbx + +# Per-database migration lists (modules to uninstall/install before a version +# bump). They describe one specific database, never a shared default, so they +# must not be versioned. Shared defaults belong to script/odoo/migration/. +odoo/ diff --git a/script/execute/execute.py b/script/execute/execute.py index 5ef684a..cfb9ff3 100644 --- a/script/execute/execute.py +++ b/script/execute/execute.py @@ -146,7 +146,16 @@ class Execute: "You cannot execute Odoo command if no version is" f" installed. Command : {redact_secrets(command)}" ) - return -1 + # Return the SAME shape the caller asked for. A bare int here + # made callers doing « status, cmd = exec_command_live(...) » + # crash with ValueError instead of seeing the failure. + if return_status_and_output_and_command: + return 1, command, [] + if return_status_and_command: + return 1, command + if return_status_and_output: + return 1, [] + return 1 command = f"source ./.venv.{source_odoo}/bin/activate && {command}" if new_window and self.cmd_source_default: command = self.cmd_source_default % command @@ -191,10 +200,16 @@ class Execute: if process.returncode != 0 and not quiet: print("Command returned error code:" f" {process.returncode}") + # An exception MUST report a failure. exit_code stays None otherwise, + # and None is falsy: callers testing « if not status: » would mark the + # step as done, and « if status and wait_at_error » would skip the error + # prompt. A crashed command was therefore recorded as a success. except FileNotFoundError: + exit_code = 1 if not quiet: print(f"Error: Command '{redact_secrets(command)}' not found.") except Exception as e: + exit_code = 1 if not quiet: print(f"An error occurred: {redact_secrets(str(e))}") process_end_time = time.time() diff --git a/script/odoo/migration/README.base.md b/script/odoo/migration/README.base.md index 7856f03..a29e31f 100644 --- a/script/odoo/migration/README.base.md +++ b/script/odoo/migration/README.base.md @@ -21,5 +21,46 @@ source ./.venv.odoo15.0_python3.8.20/bin/activate && cat ./script/odoo/migration Check [uninstall_module_list_odoo140_to_odoo150.txt](uninstall_module_list_odoo140_to_odoo150.txt) +## Module lists to uninstall + +Before a version bump, the migration uninstalls the modules listed in +`uninstall_module_list_odoo_to_odoo.txt`. Two locations are read, the +private one first, and the results are merged (duplicates dropped): + +1. `private/odoo/migration//uninstall_module_list_odooXX0_to_odooYY0.txt` + — specific to ONE database, not versioned. Which modules must be dropped + depends on the data, so this is where nearly every entry belongs. +2. `script/odoo/migration/uninstall_module_list_odooXX0_to_odooYY0.txt` + — shared defaults, versioned, valid for every database. + +Syntax: one module per line, with a justification after `#`. Commas and several +names per line are accepted; blank lines and full-line comments are ignored. +A module without a stated reason is flagged at runtime: removing a module is a +decision someone must be able to review later. + Consultez [uninstall_module_list_odoo140_to_odoo150.txt](uninstall_module_list_odoo140_to_odoo150.txt) + +## Listes de modules à désinstaller + +Avant une montée de version, la migration désinstalle les modules listés dans +`uninstall_module_list_odoo_to_odoo.txt`. Deux emplacements sont +lus, le privé d'abord, puis fusionnés (doublons éliminés) : + +1. `private/odoo/migration//uninstall_module_list_odooXX0_to_odooYY0.txt` + — propre à UNE base de données, non versionné. Les modules à supprimer + dépendent des données : c'est ici que va la quasi-totalité des entrées. +2. `script/odoo/migration/uninstall_module_list_odooXX0_to_odooYY0.txt` + — valeurs par défaut partagées, versionnées, valables pour toute base. + +Syntaxe : un module par ligne, avec une justification après `#`. Les virgules et +plusieurs noms par ligne sont acceptés ; lignes vides et commentaires pleine +ligne sont ignorés. Un module sans raison est signalé à l'exécution : supprimer +un module est une décision qui doit pouvoir être relue plus tard. + + +``` +queue_job # blocks 12->13, trigger queue_job_notify +mgmtsystem_hazard # not ported to 13.0 +web_syncer # dropped upstream +``` diff --git a/script/odoo/migration/README.fr.md b/script/odoo/migration/README.fr.md index 19ca316..35961eb 100644 --- a/script/odoo/migration/README.fr.md +++ b/script/odoo/migration/README.fr.md @@ -7,4 +7,27 @@ Exécutez ce script lors de la migration de base de données. Exemple : source ./.venv.odoo15.0_python3.8.20/bin/activate && cat ./script/odoo/migration/fix_migration_odoo140_to_odoo150.py | ./odoo15.0/odoo/odoo-bin shell -d DATABASE ``` -Consultez [uninstall_module_list_odoo140_to_odoo150.txt](uninstall_module_list_odoo140_to_odoo150.txt) \ No newline at end of file +Consultez [uninstall_module_list_odoo140_to_odoo150.txt](uninstall_module_list_odoo140_to_odoo150.txt) + +## Listes de modules à désinstaller + +Avant une montée de version, la migration désinstalle les modules listés dans +`uninstall_module_list_odoo_to_odoo.txt`. Deux emplacements sont +lus, le privé d'abord, puis fusionnés (doublons éliminés) : + +1. `private/odoo/migration//uninstall_module_list_odooXX0_to_odooYY0.txt` + — propre à UNE base de données, non versionné. Les modules à supprimer + dépendent des données : c'est ici que va la quasi-totalité des entrées. +2. `script/odoo/migration/uninstall_module_list_odooXX0_to_odooYY0.txt` + — valeurs par défaut partagées, versionnées, valables pour toute base. + +Syntaxe : un module par ligne, avec une justification après `#`. Les virgules et +plusieurs noms par ligne sont acceptés ; lignes vides et commentaires pleine +ligne sont ignorés. Un module sans raison est signalé à l'exécution : supprimer +un module est une décision qui doit pouvoir être relue plus tard. + +``` +queue_job # blocks 12->13, trigger queue_job_notify +mgmtsystem_hazard # not ported to 13.0 +web_syncer # dropped upstream +``` \ No newline at end of file diff --git a/script/odoo/migration/README.md b/script/odoo/migration/README.md index 62bf9c7..9002c6c 100644 --- a/script/odoo/migration/README.md +++ b/script/odoo/migration/README.md @@ -8,3 +8,26 @@ source ./.venv.odoo15.0_python3.8.20/bin/activate && cat ./script/odoo/migration ``` Check [uninstall_module_list_odoo140_to_odoo150.txt](uninstall_module_list_odoo140_to_odoo150.txt) + +## Module lists to uninstall + +Before a version bump, the migration uninstalls the modules listed in +`uninstall_module_list_odoo_to_odoo.txt`. Two locations are read, the +private one first, and the results are merged (duplicates dropped): + +1. `private/odoo/migration//uninstall_module_list_odooXX0_to_odooYY0.txt` + — specific to ONE database, not versioned. Which modules must be dropped + depends on the data, so this is where nearly every entry belongs. +2. `script/odoo/migration/uninstall_module_list_odooXX0_to_odooYY0.txt` + — shared defaults, versioned, valid for every database. + +Syntax: one module per line, with a justification after `#`. Commas and several +names per line are accepted; blank lines and full-line comments are ignored. +A module without a stated reason is flagged at runtime: removing a module is a +decision someone must be able to review later. + +``` +queue_job # blocks 12->13, trigger queue_job_notify +mgmtsystem_hazard # not ported to 13.0 +web_syncer # dropped upstream +``` \ No newline at end of file diff --git a/script/odoo/migration/check_cow_views.py b/script/odoo/migration/check_cow_views.py new file mode 100755 index 0000000..07321a0 --- /dev/null +++ b/script/odoo/migration/check_cow_views.py @@ -0,0 +1,347 @@ +#!/usr/bin/env python3 +# © 2021-2026 TechnoLibre (http://www.technolibre.ca) +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl) + +"""Predict which website COW views will break on the next version bump. + +Background +---------- +When a website view is customized, Odoo makes a copy-on-write (COW) copy tied +to a website_id. That copy freezes the arch AND the structure of the module +view it was copied from. + +A module view can change structure between two Odoo versions. Example measured +on a real 12.0 database: ``portal.frontend_layout`` is declared ``primary`` in +12.0 (a full QWeb template) and becomes an ``extension`` in 13.0 +(``inherit_id="web.frontend_layout"`` + xpath). During the upgrade the COW copy +follows the module and becomes an extension, but keeps its 12.0 full-template +arch. Odoo then applies the ```` root as an inheritance spec, +cannot find it in the parent, and the whole upgrade stops on:: + + ValueError: Element '' + cannot be located in parent view + +So the rule is: + + a COW view breaks when the shape its arch must have changes between + version N and version N+1. + +The discriminant is NOT ``mode``. What decides the required shape is whether +the target declares an ``inherit_id``: if it does, the arch must be inheritance +specs (````, ````, ``position=``); if it does not, the arch must be +a standalone template. Comparing ``mode`` alone misses a real case: a view +moving from a root template to ``inherit_id`` + ``primary="True"`` keeps +``mode='primary'`` on both sides yet still has to change shape, and the copy +still breaks. + +That is predictable *before* starting a multi-hour migration: the stored arch is +in the database, and the required shape is declared in the target version +sources. This script compares the two and reports the views at risk. + +It only reads: no database write, no source modification. +""" + +import argparse +import glob +import json +import os +import re +import subprocess +import sys +import xml.etree.ElementTree as ET + +# A view whose module counterpart cannot be found at all. +MODE_UNKNOWN = "unknown" + +# arch_db is text up to 15.0 and jsonb from 16.0 ({"en_US": "..."}). +RE_XML_DECLARATION = re.compile(r"<\?xml.*?\?>", re.DOTALL) +RE_FIRST_TAG = re.compile(r"<\s*([A-Za-z_][\w.:-]*)") +# Tags that carry inheritance specs rather than a standalone template. +SPEC_ROOT_TAG = ("data", "xpath") + + +def query_cow_views(database): + """Return [(id, key, mode, website_id, arch)] for every website COW view.""" + sql = ( + "SELECT id, COALESCE(key, ''), mode, website_id," + " replace(left(COALESCE(arch_db::text, ''), 400), chr(10), ' ')" + " FROM ir_ui_view WHERE website_id IS NOT NULL ORDER BY id;" + ) + result = subprocess.run( + ["psql", "-d", database, "-tAF", "|", "-c", sql], + capture_output=True, + text=True, + ) + if result.returncode: + raise RuntimeError( + f"Cannot read views from '{database}': {result.stderr.strip()}" + ) + lst_view = [] + for line in result.stdout.splitlines(): + if not line.strip(): + continue + view_id, key, mode, website_id, arch = line.split("|", 4) + lst_view.append((int(view_id), key, mode, website_id, arch)) + return lst_view + + +def arch_is_inheritance_spec(arch): + """True when the arch holds inheritance specs, not a standalone template. + + This is the real discriminant, not ``mode``. A view declared with an + ``inherit_id`` must hold specs (````, ````, or an element with + a ``position``); a root view holds a full template (````, + ``
``, ...). A copy that keeps the wrong form for what the target + version expects is exactly what raises « cannot be located in parent view ». + """ + if not arch: + return None + # From 16.0 arch_db is jsonb: take any translation, the structure is shared. + if arch.lstrip().startswith("{"): + try: + translations = json.loads(arch) + arch = next(iter(translations.values()), "") + except (ValueError, StopIteration): + # Truncated jsonb: fall through and look at the raw text. + pass + arch = RE_XML_DECLARATION.sub("", arch or "") + match = RE_FIRST_TAG.search(arch) + if not match: + return None + if match.group(1).lower() in SPEC_ROOT_TAG: + return True + # style specs. + return "position=" in arch[: match.end() + 200] + + +def load_renamed_modules(odoo_version): + """Return {old_module: new_module} from the target OpenUpgrade apriori.py. + + Without this a module renamed upstream looks absent, and every view of that + module is misreported as « module gone » instead of being checked. + """ + pattern = os.path.join(odoo_version, "**", "apriori.py") + for file_path in sorted(glob.glob(pattern, recursive=True)): + data_vars = {} + try: + with open(file_path, "r", encoding="utf-8") as f: + exec(f.read(), data_vars) # noqa: S102 - upstream data file + except Exception: + # A broken or exotic apriori.py must not stop the whole report. + continue + renamed = data_vars.get("renamed_modules") + if isinstance(renamed, dict) and renamed: + return renamed + return {} + + +def find_module_dir(odoo_version, module_name): + """Locate a module directory inside an odoo tree.""" + lst_pattern = [ + os.path.join(odoo_version, "odoo", "addons", module_name), + os.path.join(odoo_version, "odoo", "odoo", "addons", module_name), + os.path.join(odoo_version, "addons", "*", module_name), + ] + for pattern in lst_pattern: + for path in sorted(glob.glob(pattern)): + if os.path.isdir(path): + return path + return None + + +def declared_view_shape(module_dir, template_id): + """Return (mode, inherits) for a view declared in the sources, else None. + + ``inherits`` is what really matters: a declared inherit_id means the arch + must be inheritance specs. ``mode`` is kept because it is still worth + reporting, but it is NOT a reliable discriminant: a view moving from a root + template to « inherit_id + primary="True" » keeps mode='primary' on both + sides while its arch shape has to change. + """ + pattern = os.path.join(module_dir, "**", "*.xml") + for file_path in sorted(glob.glob(pattern, recursive=True)): + try: + root = ET.parse(file_path).getroot() + except ET.ParseError: + continue + for element in root.iter(): + if element.get("id") != template_id: + continue + if element.tag == "template": + inherits = bool(element.get("inherit_id")) + if str(element.get("primary", "")).lower() in ("true", "1"): + return "primary", inherits + return ("extension" if inherits else "primary"), inherits + if ( + element.tag == "record" + and element.get("model") == "ir.ui.view" + ): + mode = None + inherits = False + for field in element.findall("field"): + if field.get("name") == "mode": + mode = (field.text or "").strip() + elif field.get("name") == "inherit_id": + inherits = True + if mode: + return mode, inherits + return ("extension" if inherits else "primary"), inherits + return None + + +def analyse(database, target_version): + """Sort COW views into three buckets by comparing with the target sources. + + - at_risk : the module view changes mode -> the copy will break + - module_absent : the module itself is gone in the target version + - no_counterpart : no module view with that id, so it is a page or a record + created from the editor. Normal, and not at risk. + """ + lst_at_risk = [] + lst_module_absent = [] + lst_no_counterpart = [] + cache_shape = {} + renamed_modules = load_renamed_modules(target_version) + + for view_id, key, mode, website_id, arch in query_cow_views(database): + if not key or "." not in key: + continue + if key not in cache_shape: + module_name, _, template_id = key.partition(".") + module_dir = find_module_dir(target_version, module_name) + if module_dir is None and module_name in renamed_modules: + module_dir = find_module_dir( + target_version, renamed_modules[module_name] + ) + if module_dir is None: + cache_shape[key] = MODE_UNKNOWN + else: + cache_shape[key] = declared_view_shape(module_dir, template_id) + shape = cache_shape[key] + + if shape == MODE_UNKNOWN: + lst_module_absent.append((view_id, key, mode, website_id)) + continue + if shape is None: + lst_no_counterpart.append((view_id, key, mode, website_id)) + continue + + target_mode, target_inherits = shape + is_spec = arch_is_inheritance_spec(arch) + + # The decisive test: the target expects inheritance specs but the copy + # holds a standalone template, or the reverse. + if is_spec is not None and target_inherits != is_spec: + reason = ( + "target inherits, copy holds a standalone template" + if target_inherits + else "target is a root view, copy holds inheritance specs" + ) + lst_at_risk.append( + (view_id, key, mode, target_mode, website_id, reason) + ) + elif target_mode != mode: + # Shape is fine but the mode moves: worth reporting, less severe. + lst_at_risk.append( + ( + view_id, + key, + mode, + target_mode, + website_id, + "mode changes, arch shape unchanged", + ) + ) + return lst_at_risk, lst_module_absent, lst_no_counterpart + + +def main(): + parser = argparse.ArgumentParser( + description=( + "Report website COW views that will break on the next Odoo" + " version bump (read-only)." + ) + ) + parser.add_argument( + "-d", "--database", required=True, help="database to inspect" + ) + parser.add_argument( + "-t", + "--target_version", + required=True, + help="target Odoo source directory, e.g. odoo13.0", + ) + parser.add_argument( + "-v", + "--verbose", + action="store_true", + help="also list the editor-made pages, which are not at risk", + ) + config = parser.parse_args() + + if not os.path.isdir(config.target_version): + print( + f"❌ Target version directory '{config.target_version}' not found." + ) + return 1 + + lst_at_risk, lst_module_absent, lst_no_counterpart = analyse( + config.database, config.target_version + ) + + if not lst_at_risk: + print( + "✅ -> No website COW view changes mode in" + f" {config.target_version}." + ) + else: + print( + f"⚠️ {len(lst_at_risk)} website COW view(s) will break when moving" + f" to {config.target_version}: the copy keeps an arch whose shape" + " no longer matches what the target module view expects." + ) + for ( + view_id, + key, + mode, + target_mode, + website_id, + reason, + ) in lst_at_risk: + print( + f" - id={view_id} website={website_id} {key}" + f" : {mode} -> {target_mode} ({reason})" + ) + print( + " Arbitrate BEFORE launching the migration. To neutralize a copy," + " rename its key (UPDATE ir_ui_view SET key='zz_cow_archive.'||key," + " active=false): an unmatched key is never paired with the module" + " view, so the copy never receives the new inherit_id. Setting" + " active=false alone is NOT enough -- an inactive copy that keeps" + " the same key still shadows the module view." + ) + + if lst_module_absent: + print( + f"ℹ {len(lst_module_absent)} COW view(s) belong to a module absent" + f" from {config.target_version}:" + ) + for view_id, key, mode, website_id in lst_module_absent: + print(f" - id={view_id} website={website_id} {key} ({mode})") + + if lst_no_counterpart: + print( + f"ℹ {len(lst_no_counterpart)} COW view(s) are pages or records made" + " from the website editor (no module view of that name): not at" + " risk." + ("" if config.verbose else " Use -v to list them.") + ) + if config.verbose: + for view_id, key, mode, website_id in lst_no_counterpart: + print(f" - id={view_id} website={website_id} {key} ({mode})") + + # Informative only: never fail the migration on a warning. + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/script/odoo/migration/fix_migration_odoo130_to_odoo140.sql b/script/odoo/migration/fix_migration_odoo130_to_odoo140.sql new file mode 100644 index 0000000..cde9856 --- /dev/null +++ b/script/odoo/migration/fix_migration_odoo130_to_odoo140.sql @@ -0,0 +1,39 @@ +-- © 2021-2026 TechnoLibre (http://www.technolibre.ca) +-- License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl) +-- +-- Odoo 13.0 -> 14.0 : hand the « group_fiscal_year » security group over to +-- the module that owns it in 14.0. +-- +-- In 13.0 the group « Allow to define fiscal years of more or less than a +-- year » is declared by the core « account » module, so the database holds it +-- as account.group_fiscal_year. In 14.0 core account no longer declares it and +-- om_account_accountant (odoomates) does. During the upgrade that module finds +-- no XML id of its own, tries to CREATE the group, and hits: +-- +-- duplicate key value violates unique constraint "res_groups_name_uniq" +-- Key (category_id, name)=(9, Allow to define fiscal years ...) already exists +-- +-- Renaming the XML id makes Odoo UPDATE the existing row instead of creating a +-- duplicate. The record id is untouched, so any user assignment, access right +-- or record rule pointing at the group survives. +-- +-- Runs through psql, not the Odoo shell: at this point the database is still +-- 13.0 and loading it with the 14.0 registry is exactly what fails. + +UPDATE ir_model_data + SET module = 'om_account_accountant' + WHERE model = 'res.groups' + AND module = 'account' + AND name = 'group_fiscal_year' + -- Only when that module is actually part of this database. + AND EXISTS ( + SELECT 1 FROM ir_module_module + WHERE name = 'om_account_accountant' + AND state IN ('installed', 'to upgrade', 'to install') + ) + -- Idempotent: do nothing if the target XML id already exists. + AND NOT EXISTS ( + SELECT 1 FROM ir_model_data + WHERE module = 'om_account_accountant' + AND name = 'group_fiscal_year' + ); diff --git a/script/odoo/migration/neutralize_cow_views.py b/script/odoo/migration/neutralize_cow_views.py new file mode 100755 index 0000000..7e32425 --- /dev/null +++ b/script/odoo/migration/neutralize_cow_views.py @@ -0,0 +1,158 @@ +#!/usr/bin/env python3 +# © 2021-2026 TechnoLibre (http://www.technolibre.ca) +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl) + +"""Neutralize the website COW views that would break a version bump. + +How it works +------------ +``key`` is the only thing that pairs a website copy with the module view it +came from. A copy whose key matches nothing is never paired, so it never +receives the new ``inherit_id``, never changes shape, and never takes part in +any view combination. Renaming the key is therefore enough to take a copy out +of the way:: + + UPDATE ir_ui_view SET key = '.' || key, active = false WHERE id = ? + +Setting ``active = false`` alone would NOT work: an inactive copy that keeps +the same key still shadows the module view. + +Nothing is deleted, so ``inherit_id ondelete='restrict'`` and the +``website_page`` foreign keys are never touched, and the 12.0 arch stays in +database as a readable archive. ``--restore`` puts everything back. + +Plain psql on purpose: this must run on a database that has not been migrated +yet, where starting an Odoo shell of the target version is not guaranteed. +""" + +import argparse +import os +import subprocess +import sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +from check_cow_views import analyse # noqa: E402 + +DEFAULT_PREFIX = "zz_cow_archive" + + +def run_psql(database, sql): + """Run a statement and return stdout, raising on failure.""" + result = subprocess.run( + ["psql", "-d", database, "-tAc", sql], + capture_output=True, + text=True, + ) + if result.returncode: + raise RuntimeError( + f"Query failed on '{database}': {result.stderr.strip()}" + ) + return result.stdout.strip() + + +def neutralize(database, lst_view_id, prefix): + """Rename the key of the given views and deactivate them.""" + if not lst_view_id: + return 0 + ids = ",".join(str(view_id) for view_id in lst_view_id) + output = run_psql( + database, + "WITH updated AS (" + f" UPDATE ir_ui_view SET key = '{prefix}.' || key, active = false" + f" WHERE id IN ({ids}) AND key NOT LIKE '{prefix}.%'" + " RETURNING 1) SELECT count(*) FROM updated;", + ) + return int(output or 0) + + +def restore(database, prefix): + """Undo a neutralization: strip the prefix and reactivate.""" + output = run_psql( + database, + "WITH updated AS (" + f" UPDATE ir_ui_view SET key = substring(key from {len(prefix) + 2})," + " active = true" + f" WHERE key LIKE '{prefix}.%'" + " RETURNING 1) SELECT count(*) FROM updated;", + ) + return int(output or 0) + + +def main(): + parser = argparse.ArgumentParser( + description=( + "Neutralize the website COW views that would break a version" + " bump, by renaming their key. Dry-run unless --apply." + ) + ) + parser.add_argument("-d", "--database", required=True) + parser.add_argument( + "-t", + "--target_version", + help="target Odoo source directory, e.g. odoo13.0", + ) + parser.add_argument( + "--prefix", + default=DEFAULT_PREFIX, + help=f"archive prefix for the key (default: {DEFAULT_PREFIX})", + ) + parser.add_argument( + "--apply", action="store_true", help="actually write to the database" + ) + parser.add_argument( + "--restore", + action="store_true", + help="undo a previous neutralization and reactivate the copies", + ) + config = parser.parse_args() + + if config.restore: + count = restore(config.database, config.prefix) + print(f"✅ -> {count} COW view(s) restored on '{config.database}'.") + return 0 + + if not config.target_version: + parser.error("--target_version is required unless --restore is used") + if not os.path.isdir(config.target_version): + print( + f"❌ Target version directory '{config.target_version}' not found." + ) + return 1 + + lst_at_risk, _, _ = analyse(config.database, config.target_version) + if not lst_at_risk: + print("✅ -> No website COW view to neutralize.") + return 0 + + print( + f"⚠️ {len(lst_at_risk)} website COW view(s) would break the bump to" + f" {config.target_version}:" + ) + lst_view_id = [] + for view_id, key, mode, target_mode, website_id, reason in lst_at_risk: + lst_view_id.append(view_id) + print( + f" - id={view_id} website={website_id} {key}" + f" : {mode} -> {target_mode} ({reason})" + ) + + if not config.apply: + print( + "ℹ Dry-run. Add --apply to rename their key to" + f" '{config.prefix}.' and deactivate them. Reversible with" + " --restore; the arch stays in database." + ) + return 0 + + count = neutralize(config.database, lst_view_id, config.prefix) + print( + f"✅ -> {count} COW view(s) neutralized (key prefixed with" + f" '{config.prefix}.', deactivated). The arch is kept as an archive;" + " use --restore to undo." + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/script/odoo/migration/reset_stale_cow_views.py b/script/odoo/migration/reset_stale_cow_views.py new file mode 100755 index 0000000..13cd44e --- /dev/null +++ b/script/odoo/migration/reset_stale_cow_views.py @@ -0,0 +1,365 @@ +#!/usr/bin/env python3 +# © 2021-2026 TechnoLibre (http://www.technolibre.ca) +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl) + +"""Find the website COW copies that have gone stale, and reset them. + +The problem +----------- +A COW copy freezes the module view it was copied from. Version after version, +the MODULE view is modernised while the copy is not — it is user data, nothing +rewrites it. The copy keeps working right up to the moment a CHILD view does an +``xpath`` onto something the module added, renamed or re-anchored:: + + Element '' + cannot be located in parent view + +Observed on ``web.layout`` during 14.0 -> 15.0: the copy dated from 12.0, where +the script tag had no ``id`` and QWeb still said ``t-raw``. It had survived +until then only because the 14.0 module xpath carried a fallback +(``//head/script[@id='…'] | //head/script[last()]``) that 15.0 removed. + +What this checks +---------------- +Not a heuristic on deprecated syntax, and not an absolute one either: the +DRIFT between a copy and its module twin. For every active child of a COW +copy, each ``xpath`` expression is resolved twice — against the module twin +and against the copy. An expression the twin can satisfy and the copy cannot +is an anchor the copy has lost. + +The comparison has to be differential. Odoo resolves an xpath against the +COMBINED arch of the whole inheritance chain, so « does not resolve in this +parent » proves nothing on its own — a child of ``website.layout`` targets +``//header``, which comes from an ancestor. + +When it bites +------------- +A reported copy is not necessarily failing right now. Odoo re-validates a COW +copy when its module twin is REWRITTEN — which is what a version bump does — +or when the page is rendered for that website. So a finding is a breakage +already present in the data, waiting for the next bump to surface it. That is +precisely when it is cheap to fix. + +Resetting +--------- +``--reset`` copies the MODULE arch over the stale copy. The previous arch is +always written to a backup file first, and the diff is printed, because a copy +can hold a genuine customisation — one line among fifty of drift. Read the +diff, reset, then re-apply what mattered as an INHERITING view rather than a +full copy, so the next version bump cannot make it stale again. + +Plain psql on purpose: this runs on databases whose Odoo registry does not +load, which is precisely when it is needed. +""" + +import argparse +import datetime +import difflib +import json +import os +import re +import subprocess +import sys + + +def run_psql(database, sql): + """Run a statement and return stdout, raising on failure.""" + result = subprocess.run( + ["psql", "-d", database, "-tAc", sql], + capture_output=True, + text=True, + ) + if result.returncode: + raise RuntimeError( + f"Query failed on '{database}': {result.stderr.strip()}" + ) + return result.stdout + + +def normalise_arch(value): + """The arch as a string, whatever the column type. + + Odoo stores ``arch_db`` as text up to 15.0 and as jsonb (one entry per + language) from 16.0. Casting to text gives ``{"en_US": "]*\bexpr=([\"'])(.*?)\1", re.S) + + +def child_xpaths(arch): + """The xpath expressions a view applies to its parent.""" + return [match.group(2) for match in XPATH_RE.finditer(arch)] + + +def require_lxml(): + """The XPath engine, or a loud failure. + + Never degrade to « everything resolves » when lxml is missing: a checker + that cannot check must not answer « all clean ». Odoo's own venvs all ship + lxml; the bare system python3 usually does not. + """ + try: + from lxml import etree + + return etree + except ImportError: + sys.exit( + "❌ lxml is required to resolve the xpath expressions.\n" + " Run this with an interpreter that has it, e.g.\n" + " ./.venv.erplibre/bin/python3 " + os.path.relpath(__file__) + ) + + +def resolve(etree, arch, expr): + """True if `expr` matches something in `arch`. + + An arch that does not parse, or an expression lxml cannot evaluate, counts + as resolvable: this tool reports views that will CERTAINLY break, and must + never invent a failure it cannot substantiate. + """ + try: + tree = etree.fromstring(arch.encode("utf-8")) + except etree.XMLSyntaxError: + return True + try: + return bool(tree.xpath(expr)) + except etree.XPathEvalError: + return True + + +def analyse(database): + """[(copy, module_twin, [(child_id, failing_expr), ...])] — the COW copies + that have DRIFTED away from their module twin. + + The test is differential, and it has to be. Odoo resolves an xpath against + the COMBINED arch of the whole inheritance chain, not against the parent's + own arch, so « does not resolve here » proves nothing on its own: a child + of ``website.layout`` legitimately targets ``//header``, which comes from + an ancestor. Comparing the copy with its module twin removes that whole + class of noise: + + resolves in the module twin, not in the copy -> the copy lost an + anchor: real drift + resolves in neither -> the anchor lives + further up the chain + resolves in both -> nothing to see + + Inactive children are skipped: Odoo never applies them, so they cannot + break a load. + """ + etree = require_lxml() + views = fetch_views(database) + module_by_key = { + v["key"]: v + for v in views.values() + if v["website_id"] is None and v["key"] + } + children = {} + for view in views.values(): + if view["inherit_id"]: + children.setdefault(view["inherit_id"], []).append(view) + + findings = [] + for view in sorted(views.values(), key=lambda v: v["id"]): + twin = module_by_key.get(view["key"]) + if view["website_id"] is None or twin is None: + continue + broken = [] + for child in children.get(view["id"], []): + if not child.get("active", True): + continue + for expr in child_xpaths(child["arch"]): + if resolve(etree, twin["arch"], expr) and not resolve( + etree, view["arch"], expr + ): + broken.append((child["id"], expr)) + if broken: + findings.append((view, twin, broken)) + return findings + + +def show_diff(module_view, cow_view): + """Module arch vs copy: what the copy would gain and lose on a reset.""" + diff = difflib.unified_diff( + module_view["arch"].splitlines(), + cow_view["arch"].splitlines(), + fromfile=f"module id={module_view['id']}", + tofile=f"cow id={cow_view['id']}", + lineterm="", + ) + for line in diff: + print(f" {line}") + + +def backup(database, cow_view, directory): + """Store the arch about to be replaced, and return the file path.""" + os.makedirs(directory, exist_ok=True) + stamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") + path = os.path.join( + directory, f"{cow_view['key']}_{cow_view['id']}_{stamp}.json" + ) + with open(path, "w", encoding="utf-8") as fh: + json.dump( + { + "database": database, + "id": cow_view["id"], + "key": cow_view["key"], + "website_id": cow_view["website_id"], + "saved_at": stamp, + "arch_db": cow_view["arch"], + }, + fh, + indent=2, + ensure_ascii=False, + ) + return path + + +def reset(database, cow_view, module_view): + """Copy the module arch over the stale copy. Returns rows updated.""" + sql = ( + "UPDATE ir_ui_view c SET arch_db = m.arch_db " + "FROM ir_ui_view m " + f"WHERE c.id = {cow_view['id']} AND m.id = {module_view['id']}" + ) + run_psql(database, sql) + return 1 + + +def main(): + parser = argparse.ArgumentParser( + description=( + "Report the website COW copies whose children can no longer be" + " applied, and optionally reset them onto the module view." + ) + ) + parser.add_argument("-d", "--database", required=True) + parser.add_argument( + "--reset", + metavar="KEY", + action="append", + default=[], + help="reset this key onto its module view ('all' for every finding)", + ) + parser.add_argument( + "--apply", + action="store_true", + help="really write; without it --reset only shows what it would do", + ) + parser.add_argument( + "--backup-dir", + default=None, + help="where the replaced arch is saved" + " (default private/odoo/migration//cow_reset)", + ) + config = parser.parse_args() + + findings = analyse(config.database) + if not findings: + print("✅ No COW copy has drifted from its module view.") + return 0 + + print( + f"⚠️ {len(findings)} COW copy(ies) drifted from their module view" + f" in {config.database}" + ) + print( + " Odoo surfaces this when the module view is rewritten (a version" + " bump) or when the page is rendered.\n" + ) + for cow_view, module_view, broken in findings: + twin = ( + f"module id={module_view['id']}" + if module_view + else "NO module view with this key" + ) + print( + f" id={cow_view['id']} key={cow_view['key']}" + f" website_id={cow_view['website_id']} [{twin}]" + ) + for child_id, expr in broken: + print(f" child {child_id} cannot apply: {expr}") + if module_view: + show_diff(module_view, cow_view) + print() + + if not config.reset: + print( + "Nothing changed. Re-run with --reset --apply to reset a" + " copy onto its module view." + ) + return 1 + + wanted = set(config.reset) + directory = config.backup_dir or os.path.join( + "private", "odoo", "migration", config.database, "cow_reset" + ) + done = 0 + for cow_view, module_view, _broken in findings: + if "all" not in wanted and cow_view["key"] not in wanted: + continue + if not module_view: + print( + f"⏭ {cow_view['key']}: no module view to reset onto," + " skipped." + ) + continue + if not config.apply: + print( + f"[dry-run] would reset id={cow_view['id']}" + f" ({cow_view['key']}) onto id={module_view['id']}" + ) + continue + path = backup(config.database, cow_view, directory) + reset(config.database, cow_view, module_view) + done += 1 + print(f"✅ reset id={cow_view['id']} ({cow_view['key']})") + print(f" previous arch saved to {path}") + if config.apply and done: + print( + f"\n{done} copy(ies) reset. Re-apply any real customisation as an" + " INHERITING view, not a copy, so it cannot go stale again." + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/script/odoo/migration/snapshot_cow_views.py b/script/odoo/migration/snapshot_cow_views.py new file mode 100755 index 0000000..a8f839b --- /dev/null +++ b/script/odoo/migration/snapshot_cow_views.py @@ -0,0 +1,213 @@ +#!/usr/bin/env python3 +# © 2021-2026 TechnoLibre (http://www.technolibre.ca) +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl) + +"""Snapshot the website COW views, and diff two snapshots. + +Why +--- +A version bump rewrites website views in ways nobody announces: OpenUpgrade +converts Bootstrap markup on every ``website_id IS NOT NULL`` view, modules +rewrite the copies they own, and some copies simply disappear. Without a +before/after record, "the site looks wrong" is unanswerable. + +Taking a snapshot before and after each jump turns that into a diff: which copy +lost its arch, which changed mode, which was renamed, which vanished. + +Snapshots hold customer template content, so they belong under ``private/`` +and are never versioned. + +Usage:: + + snapshot_cow_views.py -d --label before_13 + snapshot_cow_views.py -d --label after_13 + snapshot_cow_views.py --diff +""" + +import argparse +import datetime +import hashlib +import json +import os +import subprocess +import sys + +# Columns worth recording. ir_ui_view does not expose the same set across 12.0 +# to 18.0, so the query keeps only those that actually exist. +WANTED_COLUMN = [ + "id", + "key", + "name", + "type", + "mode", + "active", + "priority", + "website_id", + "inherit_id", + "arch_fs", + "arch_updated", +] +DEFAULT_DIR = os.path.join("private", "odoo", "migration") + + +def run_psql(database, sql): + """Run a read-only query and return stdout.""" + result = subprocess.run( + ["psql", "-d", database, "-tAc", sql], + capture_output=True, + text=True, + ) + if result.returncode: + raise RuntimeError( + f"Query failed on '{database}': {result.stderr.strip()}" + ) + return result.stdout + + +def existing_columns(database): + """Column names of ir_ui_view present in this database.""" + output = run_psql( + database, + "SELECT column_name FROM information_schema.columns" + " WHERE table_name = 'ir_ui_view';", + ) + return {line.strip() for line in output.splitlines() if line.strip()} + + +def collect(database): + """Return the list of COW views as plain dicts, arch included. + + The rows come back as JSON straight from Postgres: an arch holds newlines + and pipes, so no hand-made separator survives it. + """ + available = existing_columns(database) + lst_column = [name for name in WANTED_COLUMN if name in available] + select = ", ".join(lst_column) + ", arch_db::text AS arch_db" + output = run_psql( + database, + "SELECT COALESCE(json_agg(row_to_json(t)), '[]'::json) FROM (" + f" SELECT {select} FROM ir_ui_view" + " WHERE website_id IS NOT NULL ORDER BY id) t;", + ) + lst_view = json.loads(output or "[]") + for view in lst_view: + arch = view.pop("arch_db", None) or "" + view["arch_md5"] = hashlib.md5(arch.encode("utf-8")).hexdigest() + view["arch_len"] = len(arch) + view["arch_db"] = arch + return lst_view + + +def save(database, label, output_dir): + """Write a snapshot and return its path.""" + directory = output_dir or os.path.join( + DEFAULT_DIR, database, "cow_snapshots" + ) + os.makedirs(directory, exist_ok=True) + lst_view = collect(database) + payload = { + "database": database, + "label": label, + "taken_at": datetime.datetime.now().isoformat(timespec="seconds"), + "count": len(lst_view), + "views": lst_view, + } + file_path = os.path.join(directory, f"{label}.json") + with open(file_path, "w", encoding="utf-8") as f: + json.dump(payload, f, indent=2, ensure_ascii=False) + print(f"✅ -> {len(lst_view)} COW view(s) recorded in {file_path}") + return file_path + + +def load(file_path): + with open(file_path, "r", encoding="utf-8") as f: + return json.load(f) + + +def diff(path_before, path_after): + """Print what changed between two snapshots.""" + before = load(path_before) + after = load(path_after) + map_before = {view["id"]: view for view in before["views"]} + map_after = {view["id"]: view for view in after["views"]} + + removed = sorted(set(map_before) - set(map_after)) + added = sorted(set(map_after) - set(map_before)) + common = sorted(set(map_before) & set(map_after)) + + print( + f"📊 {before.get('label')} ({before.get('count')} views)" + f" -> {after.get('label')} ({after.get('count')} views)" + ) + + if removed: + print(f"❌ {len(removed)} COW view(s) disappeared:") + for view_id in removed: + view = map_before[view_id] + print(f" - id={view_id} {view.get('key')}") + + if added: + print(f"➕ {len(added)} COW view(s) appeared:") + for view_id in added: + view = map_after[view_id] + print(f" - id={view_id} {view.get('key')} ({view.get('mode')})") + + lst_changed = [] + for view_id in common: + old, new = map_before[view_id], map_after[view_id] + lst_field = [] + for field in ("key", "mode", "inherit_id", "active", "arch_md5"): + if old.get(field) != new.get(field): + if field == "arch_md5": + lst_field.append( + f"arch rewritten ({old.get('arch_len')} ->" + f" {new.get('arch_len')} chars)" + ) + else: + lst_field.append( + f"{field}: {old.get(field)} -> {new.get(field)}" + ) + if lst_field: + lst_changed.append((view_id, new.get("key"), lst_field)) + + if lst_changed: + print(f"✏️ {len(lst_changed)} COW view(s) changed:") + for view_id, key, lst_field in lst_changed: + print(f" - id={view_id} {key}") + for change in lst_field: + print(f" {change}") + + if not (removed or added or lst_changed): + print("✅ -> No change on the website COW views.") + return 0 + + +def main(): + parser = argparse.ArgumentParser( + description="Snapshot website COW views, or diff two snapshots." + ) + parser.add_argument("-d", "--database", help="database to snapshot") + parser.add_argument( + "-l", "--label", help="snapshot name, e.g. before_13 or after_13" + ) + parser.add_argument( + "-o", "--output_dir", help="where to write (default: private/...)" + ) + parser.add_argument( + "--diff", + nargs=2, + metavar=("BEFORE", "AFTER"), + help="compare two snapshot files instead of taking one", + ) + config = parser.parse_args() + + if config.diff: + return diff(*config.diff) + if not config.database or not config.label: + parser.error("--database and --label are required to take a snapshot") + save(config.database, config.label, config.output_dir) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/script/todo/migration_form.py b/script/todo/migration_form.py new file mode 100644 index 0000000..98a150c --- /dev/null +++ b/script/todo/migration_form.py @@ -0,0 +1,186 @@ +#!/usr/bin/env python3 +# © 2021-2026 TechnoLibre (http://www.technolibre.ca) +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl) +"""Écran de reprise de migration Odoo, en TUI. + +Pendant de `qemu_deploy_form` pour l'outil de migration. Deux interfaces +posent la MÊME première question — « où en est-on, et par où reprend-on ? » — +et renvoient les MÊMES chaînes de réponse (« c », « n », « r », « q », +« 0 »..« 4 », « 4. ») que `TodoUpgrade.apply_resume_answer` traduit +en progression. La décision est donc écrite une seule fois. + +- run_resume_tui(ctx, run_app=True) : renvoie la réponse, ou None pour + retomber sur les invites en ligne. + +`ctx` vient de `TodoUpgrade.resume_context()` : pure donnée, aucun accès à la +base ni au disque depuis l'affichage. +""" +from __future__ import annotations + +try: + from script.todo.todo_i18n import t +except Exception: # pragma: no cover - repli si i18n indisponible + + def t(key: str) -> str: + return key + + +def version_line(versions): + """« 13✓ 14✓ 15 16 17 18 » — l'avancement des montées de version.""" + return " ".join( + f"{v['version']}{'✓' if v['done'] else ''}" for v in versions + ) + + +def next_version(versions): + """Première version pas encore migrée : celle que « continuer » reprend.""" + for item in versions: + if not item["done"]: + return item["version"] + return None + + +def run_resume_tui(ctx, run_app: bool = True): + """Écran de reprise. Renvoie la réponse choisie, ou None si annulé. + `run_app=False` renvoie l'instance sans la lancer (tests headless).""" + from textual.app import App, ComposeResult + from textual.containers import Horizontal, Vertical + from textual.widgets import ( + Button, + DataTable, + Footer, + Header, + OptionList, + Static, + ) + from textual.widgets.option_list import Option + + result = {"answer": None} + + class Resume(App): + CSS = """ + #head { height: auto; padding: 0 1; color: $text-muted; } + #steps { height: auto; max-height: 12; border: solid $accent; } + #bumps { height: auto; max-height: 10; border: solid $panel; } + .grouptitle { color: $accent; text-style: bold; padding: 1 1 0 1; } + #actions { height: auto; padding: 1 1 0 1; } + #hint { height: auto; color: $text-muted; padding: 0 1; } + """ + BINDINGS = [ + ("c", "cont", t("Continue where it stopped")), + ("n", "new", t("New migration, erase everything")), + ("r", "keep_zip", t("Keep the zip only")), + ("q", "quit_nothing", t("Quit without doing anything")), + ("escape", "quit_nothing", t("Quit without doing anything")), + ] + + def compose(self) -> ComposeResult: + yield Header() + yield Static( + f" {t('File'):<9}: {ctx['file']}\n" + f" {t('Database'):<9}: {ctx['database']}" + f" · {t('Target')} : {ctx['target']}\n" + f" {t('Started'):<9}: {ctx['started']}", + id="head", + ) + yield Static(f"{t('Steps')}", classes="grouptitle") + yield DataTable(id="steps") + if ctx["versions"]: + yield Static( + f"{t('Version bumps')} ({version_line(ctx['versions'])})", + classes="grouptitle", + ) + yield OptionList(id="bumps") + with Vertical(): + with Horizontal(id="actions"): + yield Button( + t("Continue where it stopped"), + variant="primary", + id="a_cont", + ) + yield Button(t("New migration"), id="a_new") + yield Button(t("Keep the zip only"), id="a_keep") + yield Button(t("Quit"), id="a_quit") + yield Static( + f" {t('Enter on a step or a version = replay from there')}", + id="hint", + ) + yield Footer() + + def on_mount(self) -> None: + self.title = t("Migration in progress") + table = self.query_one("#steps", DataTable) + table.cursor_type = "row" + table.add_columns("", "", t("Step"), t("Detail")) + for item in ctx["steps"]: + table.add_row( + f"[{item['step']}]", + item["icon"], + item["label"], + item["detail"], + ) + # Curseur sur la première étape inachevée : c'est là que ça a + # calé, donc là qu'on veut probablement rejouer. + for index, item in enumerate(ctx["steps"]): + if item["icon"] != "✅": + table.move_cursor(row=index) + break + if ctx["versions"]: + bumps = self.query_one("#bumps", OptionList) + for item in ctx["versions"]: + mark = "✓" if item["done"] else " " + bumps.add_option( + Option( + f" {mark} Odoo {item['version']}.0 — " + f"{t('rebuilds the intermediate database')}", + id=str(item["version"]), + ) + ) + upcoming = next_version(ctx["versions"]) + if upcoming is not None: + bumps.highlighted = [ + v["version"] for v in ctx["versions"] + ].index(upcoming) + + # -- choix ------------------------------------------------------ # + def _answer(self, value): + result["answer"] = value + self.exit() + + def on_data_table_row_selected(self, event) -> None: + index = event.cursor_row + if 0 <= index < len(ctx["steps"]): + self._answer(str(ctx["steps"][index]["step"])) + + def on_option_list_option_selected(self, event) -> None: + self._answer(f"4.{event.option.id}") + + def on_button_pressed(self, event) -> None: + mapping = { + "a_cont": "c", + "a_new": "n", + "a_keep": "r", + "a_quit": "q", + } + value = mapping.get(event.button.id) + if value: + self._answer(value) + + def action_cont(self) -> None: + self._answer("c") + + def action_new(self) -> None: + self._answer("n") + + def action_keep_zip(self) -> None: + self._answer("r") + + def action_quit_nothing(self) -> None: + self._answer("q") + + app = Resume() + app._result = result # lecture par les tests headless + if not run_app: + return app + app.run() + return result["answer"] diff --git a/script/todo/migration_stats.py b/script/todo/migration_stats.py new file mode 100644 index 0000000..715d7cc --- /dev/null +++ b/script/todo/migration_stats.py @@ -0,0 +1,201 @@ +#!/usr/bin/env python3 +# © 2021-2026 TechnoLibre (http://www.technolibre.ca) +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl) +"""Statistiques d'une migration Odoo, en lecture seule. + +Ne touche NI la base NI le fichier de progression : tout se déduit du journal +de migration et des traces laissées sous `private/odoo/migration//`. +C'est ce qui permet de consulter l'état d'une migration en cours depuis une +autre session sans risquer de la perturber. + +`compute()` reçoit le contexte déjà construit par +`TodoUpgrade.resume_context()` — étapes et montées de version — pour ne pas +réimplémenter une seconde fois la lecture des clés « state_* », qui +divergerait de l'écran de reprise. +""" +from __future__ import annotations + +import datetime +import glob +import json +import os + +try: + from script.todo.todo_i18n import t +except Exception: # pragma: no cover - repli si i18n indisponible + + def t(key: str) -> str: + return key + + +def _as_int(value): + """Clé de version en entier. JSON transforme les clés numériques en + chaînes : « 13 » et 13 désignent la même version.""" + try: + return int(float(value)) + except (TypeError, ValueError): + return None + + +def fmt_delay(start, end): + """Écart lisible entre deux horodatages « str(datetime) ».""" + try: + a = datetime.datetime.fromisoformat(str(start)) + b = datetime.datetime.fromisoformat(str(end)) + except (TypeError, ValueError): + return "?" + secs = int(abs((b - a).total_seconds())) + days, rest = divmod(secs, 86400) + hours, rest = divmod(rest, 3600) + minutes = rest // 60 + if days: + return f"{days} j {hours:02d} h" + if hours: + return f"{hours} h {minutes:02d} min" + return f"{minutes} min" + + +def module_evolution(dct_progression): + """[(version, nb_modules, delta)] du plus ancien au plus récent. + + Montre où les modules disparaissent : un saut qui en perd 15 d'un coup + n'a pas le même sens qu'un saut qui n'en perd aucun. + """ + raw = dct_progression.get("dct_module_per_version") or {} + rows = [] + for key, value in raw.items(): + version = _as_int(key) + if version is not None and isinstance(value, list): + rows.append((version, len(value))) + rows.sort() + out = [] + previous = None + for version, count in rows: + out.append( + (version, count, None if previous is None else count - previous) + ) + previous = count + return out + + +def cow_snapshots(private_dir, database_name): + """Instantanés de vues COW enregistrés, du plus ancien au plus récent.""" + directory = os.path.join(private_dir, database_name, "cow_snapshots") + out = [] + for path in sorted(glob.glob(os.path.join(directory, "*.json"))): + try: + with open(path, encoding="utf-8") as fh: + data = json.load(fh) + except (OSError, ValueError): + continue + out.append( + { + "label": data.get("label") or os.path.basename(path)[:-5], + "count": data.get("count"), + "taken_at": data.get("taken_at") or "?", + "path": path, + } + ) + out.sort(key=lambda item: item["taken_at"]) + return out + + +def fix_hooks(ctx, global_dir): + """Correctifs de migration disponibles, et lesquels ont tourné.""" + applied = ctx.get("_fix_applied") or [] + out = [] + for index, item in enumerate(ctx.get("versions") or []): + target = item["version"] + stem = f"fix_migration_odoo{(target - 1) * 10}_to_odoo{target * 10}" + found = [ + os.path.basename(p) + for ext in (".sql", ".py") + for p in [os.path.join(global_dir, stem + ext)] + if os.path.exists(p) + ] + if not found: + continue + out.append( + { + "version": target, + "file": found[0], + "applied": bool(index < len(applied) and applied[index]), + } + ) + return out + + +def journal(dct_progression): + """Commandes exécutées et décisions annotées (les lignes « # »).""" + lst = dct_progression.get("command_executed") or [] + comments = [ + c[2:].strip() for c in lst if isinstance(c, str) and c.startswith("# ") + ] + commands = [ + c for c in lst if isinstance(c, str) and not c.startswith("# ") + ] + return {"commands": commands, "comments": comments} + + +def compute( + dct_progression, + ctx, + database_name, + read_uninstall, + private_dir, + global_dir, +): + """Rassemble toutes les statistiques. + + `read_uninstall(version, base)` est la lecture de liste de TodoUpgrade : + elle résout ELLE-MÊME ses chemins privé puis global, ce qui garantit + qu'on affiche exactement la liste qui serait appliquée. `private_dir` ne + sert donc qu'aux instantanés COW, et `global_dir` qu'aux correctifs.""" + versions = [item["version"] for item in (ctx.get("versions") or [])] + uninstall = {} + for target in versions: + try: + _lst, detail = read_uninstall(target - 1, database_name) + except Exception: + detail = [] + if detail: + uninstall[target] = detail + + evolution = module_evolution(dct_progression) + origin = dct_progression.get("lst_module_per_version_origin") or [] + return { + "delay": fmt_delay( + dct_progression.get("date_create"), + dct_progression.get("date_update"), + ), + "updated": dct_progression.get("date_update") or "?", + "evolution": evolution, + "origin_count": len(origin) if isinstance(origin, list) else 0, + "missing": dct_progression.get("lst_module_missing") or [], + "duplicate": dct_progression.get("lst_module_duplicate") or [], + "uninstall": uninstall, + "removed_total": sum(len(v) for v in uninstall.values()), + "cow": cow_snapshots(private_dir, database_name), + "fixes": fix_hooks( + dict( + ctx, + _fix_applied=dct_progression.get( + "state_4_fix_migration_odoo_lst" + ) + or [], + ), + global_dir, + ), + "journal": journal(dct_progression), + } + + +def flat_module_list(uninstall): + """Tous les modules supprimés, dédupliqués, prêts à copier-coller.""" + seen = [] + for detail in uninstall.values(): + for item in detail: + name = item[0] if isinstance(item, (list, tuple)) else item + if name not in seen: + seen.append(name) + return seen diff --git a/script/todo/todo_i18n.py b/script/todo/todo_i18n.py index e17e94f..5b8eba3 100644 --- a/script/todo/todo_i18n.py +++ b/script/todo/todo_i18n.py @@ -1687,6 +1687,200 @@ TRANSLATIONS = { "fr": "via", "en": "via", }, + "Odoo migration interface": { + "fr": "🚚 Interface de la migration Odoo", + "en": "🚚 Odoo migration interface", + }, + "modules of the list have no code in the active Odoo:": { + "fr": "modules de la liste n'ont plus de code dans l'Odoo actif :", + "en": "modules of the list have no code in the active Odoo:", + }, + "Odoo cannot uninstall a module whose code is gone;": { + "fr": "Odoo ne peut pas désinstaller un module dont le code a" + " disparu ;", + "en": "Odoo cannot uninstall a module whose code is gone;", + }, + "one of them fails the whole uninstall command.": { + "fr": "un seul d'entre eux fait échouer toute la désinstallation.", + "en": "one of them fails the whole uninstall command.", + }, + "are present and can be uninstalled:": { + "fr": "sont présents et peuvent être désinstallés :", + "en": "are present and can be uninstalled:", + }, + "No module of the list is present.": { + "fr": "Aucun module de la liste n'est présent.", + "en": "No module of the list is present.", + }, + "Uninstall the present ones, skip the missing": { + "fr": "Désinstaller les présents, ignorer les manquants", + "en": "Uninstall the present ones, skip the missing", + }, + "Try the whole list anyway (it will fail)": { + "fr": "Tout tenter quand même (ça échouera)", + "en": "Try the whole list anyway (it will fail)", + }, + "Uninstall nothing, continue": { + "fr": "Ne rien désinstaller, continuer", + "en": "Uninstall nothing, continue", + }, + "Nothing uninstalled.": { + "fr": "Rien de désinstallé.", + "en": "Nothing uninstalled.", + }, + "Check the COW views that drifted": { + "fr": "Vérifier les vues COW en retard sur leur vue module", + "en": "Check the COW views that drifted", + }, + "Tool not found": { + "fr": "Outil introuvable", + "en": "Tool not found", + }, + "To reset one of them onto its module view:": { + "fr": "Pour réinitialiser l'une d'elles sur sa vue module :", + "en": "To reset one of them onto its module view:", + }, + "Read the diff first: a copy can hold a customisation.": { + "fr": "Lire le diff d'abord : une copie peut porter une" + " personnalisation.", + "en": "Read the diff first: a copy can hold a customisation.", + }, + "Migration statistics (read-only)": { + "fr": "📊 Statistiques de la migration (lecture seule)", + "en": "📊 Migration statistics (read-only)", + }, + "Choice (0-3, default 1): ": { + "fr": "Choix (0-3, défaut 1) : ", + "en": "Choice (0-3, default 1): ", + }, + "Migration statistics": { + "fr": "Statistiques de la migration", + "en": "Migration statistics", + }, + "elapsed": { + "fr": "écoulé", + "en": "elapsed", + }, + "Level reached": { + "fr": "Niveau atteint", + "en": "Level reached", + }, + "Modules": { + "fr": "Modules", + "en": "Modules", + }, + "modules": { + "fr": "modules", + "en": "modules", + }, + "At the start": { + "fr": "Au départ", + "en": "At the start", + }, + "Removed in total": { + "fr": "Supprimés au total", + "en": "Removed in total", + }, + "Reported missing": { + "fr": "Signalés manquants", + "en": "Reported missing", + }, + "Duplicated": { + "fr": "En double", + "en": "Duplicated", + }, + "Migration fixes": { + "fr": "Correctifs de migration", + "en": "Migration fixes", + }, + "COW views": { + "fr": "Vues COW", + "en": "COW views", + }, + "views": { + "fr": "vues", + "en": "views", + }, + "no snapshot": { + "fr": "aucun instantané", + "en": "no snapshot", + }, + "Journal": { + "fr": "Journal", + "en": "Journal", + }, + "commands": { + "fr": "commandes", + "en": "commands", + }, + "recorded decisions": { + "fr": "décisions annotées", + "en": "recorded decisions", + }, + "Removed modules, with their reason": { + "fr": "Modules supprimés, avec leur justification", + "en": "Removed modules, with their reason", + }, + "Removed modules, comma-separated (copy)": { + "fr": "Modules supprimés, séparés par des virgules (à copier)", + "en": "Removed modules, comma-separated (copy)", + }, + "COW views: snapshots and differences": { + "fr": "Vues COW : instantanés et différences", + "en": "COW views: snapshots and differences", + }, + "Recorded decisions (journal)": { + "fr": "Décisions annotées (journal)", + "en": "Recorded decisions (journal)", + }, + "Executed commands (last 30)": { + "fr": "Commandes exécutées (30 dernières)", + "en": "Executed commands (last 30)", + }, + "nothing recorded": { + "fr": "rien d'annoté", + "en": "nothing recorded", + }, + "Need two snapshots to diff.": { + "fr": "Il faut deux instantanés pour comparer.", + "en": "Need two snapshots to diff.", + }, + "Diff which two? (e.g. 1,2 — blank to skip)": { + "fr": "Comparer lesquels ? (ex. 1,2 — vide pour passer)", + "en": "Diff which two? (e.g. 1,2 — blank to skip)", + }, + "No migration in progress to resume.": { + "fr": "Aucune migration en cours à reprendre.", + "en": "No migration in progress to resume.", + }, + "Quit without doing anything": { + "fr": "Quitter sans rien faire", + "en": "Quit without doing anything", + }, + "Version bumps": { + "fr": "Montées de version", + "en": "Version bumps", + }, + "Step": { + "fr": "Étape", + "en": "Step", + }, + "Detail": { + "fr": "Détail", + "en": "Detail", + }, + "Enter on a step or a version = replay from there": { + "fr": "Entrée sur une étape ou une version = rejouer depuis là", + "en": "Enter on a step or a version = replay from there", + }, + "New migration": { + "fr": "Nouvelle migration", + "en": "New migration", + }, + "Keep the zip only": { + "fr": "Garder seulement le zip", + "en": "Keep the zip only", + }, "Reset all preferences": { "fr": "🧹 Réinitialiser toutes les préférences", "en": "🧹 Reset all preferences", @@ -1952,6 +2146,26 @@ TRANSLATIONS = { "fr": "Échec de la commande : ", "en": "Command failed: ", }, + "Textual is required for this screen.": { + "fr": "Textual est nécessaire pour cet écran.", + "en": "Textual is required for this screen.", + }, + "Install it now? (Y/n): ": { + "fr": "L'installer maintenant ? (O/n, défaut : oui) : ", + "en": "Install it now? (Y/n, default: yes): ", + }, + "Textual is installed.": { + "fr": "Textual est installé.", + "en": "Textual is installed.", + }, + "Installation finished but textual is still missing.": { + "fr": "Installation terminée mais textual reste introuvable.", + "en": "Installation finished but textual is still missing.", + }, + "pip exited with": { + "fr": "pip a retourné", + "en": "pip exited with", + }, "Your distribution may package it as python3-textual.": { "fr": "Votre distribution le fournit peut-être en python3-textual.", "en": "Your distribution may package it as python3-textual.", @@ -3125,6 +3339,85 @@ TRANSLATIONS = { "recorded runs": {"fr": "runs enregistrés", "en": "recorded runs"}, "runs erased": {"fr": "runs effacés", "en": "runs erased"}, "Cancelled.": {"fr": "Annulé.", "en": "Cancelled."}, + # Database migration - resume menu + "Migration in progress": { + "fr": "Migration en cours", + "en": "Migration in progress", + }, + "File": {"fr": "Fichier", "en": "File"}, + "Database": {"fr": "Base", "en": "Database"}, + "Target": {"fr": "Cible", "en": "Target"}, + "Started": {"fr": "Démarrée", "en": "Started"}, + "Steps": {"fr": "Étapes", "en": "Steps"}, + "Prepare the environment": { + "fr": "Préparer l'environnement", + "en": "Prepare the environment", + }, + "Restore and neutralize the database": { + "fr": "Restaurer et neutraliser la base", + "en": "Restore and neutralize the database", + }, + "Update all addons": { + "fr": "Mettre à jour tous les modules", + "en": "Update all addons", + }, + "Clean up before data migration": { + "fr": "Nettoyer avant la migration des données", + "en": "Clean up before data migration", + }, + "Upgrade version by version (OpenUpgrade)": { + "fr": "Monter de version en version (OpenUpgrade)", + "en": "Upgrade version by version (OpenUpgrade)", + }, + "not started": {"fr": "non démarrée", "en": "not started"}, + "done": {"fr": "terminée", "en": "done"}, + "partially done": {"fr": "partielle", "en": "partially done"}, + "version bumps migrated": { + "fr": "montées de version faites", + "en": "version bumps migrated", + }, + "Continue where it stopped": { + "fr": "Continuer là où ça s'est arrêté", + "en": "Continue where it stopped", + }, + "Replay from that step": { + "fr": "Reprendre à partir de cette étape", + "en": "Replay from that step", + }, + "erases the progression of that step and the next ones": { + "fr": "efface la progression de cette étape et des suivantes", + "en": "erases the progression of that step and the next ones", + }, + "New migration, erase everything": { + "fr": "Nouvelle migration, tout effacer", + "en": "New migration, erase everything", + }, + "Keep the zip only, ask every question again": { + "fr": "Garder seulement le zip, reposer toutes les questions", + "en": "Keep the zip only, ask every question again", + }, + "Your choice": {"fr": "Votre choix", "en": "Your choice"}, + "Unknown choice, continuing where it stopped": { + "fr": "Choix inconnu, on continue là où ça s'est arrêté", + "en": "Unknown choice, continuing where it stopped", + }, + "Replaying from step": { + "fr": "Reprise à partir de l'étape", + "en": "Replaying from step", + }, + "Replay the upgrade from version N": { + "fr": "Reprendre la montée à partir de la version N", + "en": "Replay the upgrade from version N", + }, + "rebuilds the intermediate database": { + "fr": "recrée la base intermédiaire", + "en": "rebuilds the intermediate database", + }, + "Unknown version": {"fr": "Version inconnue", "en": "Unknown version"}, + "The progression file is invalid, ignoring it": { + "fr": "Le fichier de progression est invalide, on l'ignore", + "en": "The progression file is invalid, ignoring it", + }, # --- Menu Execute › Analyse --- # L'émoji vit dans la valeur, jamais dans la clé. Deux clés distinctes, # comme « Data » / « Database » : l'entrée de menu porte l'émoji, diff --git a/script/todo/todo_upgrade.py b/script/todo/todo_upgrade.py index d77b6dc..6c37836 100755 --- a/script/todo/todo_upgrade.py +++ b/script/todo/todo_upgrade.py @@ -6,15 +6,25 @@ import datetime import json import logging import os +import re import shutil import sys import zipfile from uuid import uuid4 -from script.todo.version_manager import get_odoo_version import click import todo_file_browser +from script.todo.version_manager import get_odoo_version + +try: + from script.todo.todo_i18n import t +except Exception: # pragma: no cover - fallback when i18n is unavailable + + def t(key: str) -> str: + return key + + new_path = os.path.normpath( os.path.join(os.path.dirname(__file__), "..", "..") ) @@ -43,6 +53,23 @@ FILENAME_ODOO_VERSION = ".odoo-version" LOCAL_MANIFEST = os.path.join( ".repo", "local_manifests", "erplibre_manifest.xml" ) +# Module lists for a version bump. The shared, versioned defaults live under +# script/; the per-database lists live under private/ (mirroring script/, like +# script/todo/todo.json -> private/todo/todo_override.json). Which modules must +# be dropped depends on the database, so that choice is never versioned. +PATH_MIGRATION_GLOBAL = os.path.join("script", "odoo", "migration") +PATH_MIGRATION_PRIVATE = os.path.join("private", "odoo", "migration") +# Steps of the migration, in order. Each one owns the progression keys prefixed +# with « state_ ». Rewinding to a step drops its keys and every later +# one, so the run replays from there. Labels go through t(): the key IS the +# English string, as everywhere else in this project. +MIGRATION_STEP = [ + (0, "Prepare the environment"), + (1, "Restore and neutralize the database"), + (2, "Update all addons"), + (3, "Clean up before data migration"), + (4, "Upgrade version by version (OpenUpgrade)"), +] class TodoUpgrade: @@ -68,6 +95,501 @@ class TodoUpgrade: with open(UPGRADE_DATABASE_CONFIG_LOG, "w") as f: json.dump(self.dct_progression, f, indent=4) + @staticmethod + def read_progression(): + """Return the saved progression, or an empty dict if unreadable.""" + try: + with open(UPGRADE_DATABASE_CONFIG_LOG, "r") as f: + return json.load(f) + except (json.decoder.JSONDecodeError, OSError): + print( + f"⚠️ {t('The progression file is invalid, ignoring it')}:" + f" {UPGRADE_DATABASE_CONFIG_LOG}" + ) + return {} + + @staticmethod + def step_status(dct_progression, step): + """Return (icon, detail) telling how far a migration step went. + + Steps 0 to 3 are plain booleans. Step 4 keeps one list per version, so + its detail reports which version bumps are already migrated. + """ + prefix = f"state_{step}_" + dct_flag = { + key: value + for key, value in dct_progression.items() + if key.startswith(prefix) + } + if not dct_flag: + return "⬜", t("not started") + + if step == 4: + lst_done = dct_progression.get("state_4_upgrade_odoo_lst") or [] + # The data migration list only appears once a bump succeeds, so the + # number of bumps comes from any per-version list (« *_odoo_lst »). + total = max( + [ + len(value) + for key, value in dct_progression.items() + if key.startswith("state_4_") + and key.endswith("_odoo_lst") + and isinstance(value, list) + ] + or [0] + ) + done = sum(1 for item in lst_done if item) + detail = f"{done}/{total} " + t("version bumps migrated") + # Name the versions when the target is known: the list ends on the + # target, so the first bump is target - total + 1. + try: + last = int(float(dct_progression["target_odoo_version"])) + detail += " · " + " ".join( + f"{last - total + 1 + i}" + f"{'✓' if i < len(lst_done) and lst_done[i] else ''}" + for i in range(total) + ) + except (KeyError, TypeError, ValueError): + pass + return ("✅" if total and done == total else "⏳"), detail + + if all(dct_flag.values()): + return "✅", t("done") + return "⏳", t("partially done") + + def resume_context(self, old_dct_progression): + """Everything the resume screen shows, as plain data. + + No I/O: the line-by-line prompt and the TUI both render THIS, so the + two can never describe the migration differently. + """ + migration_file = old_dct_progression.get("migration_file") or "?" + steps = [] + for step, label in MIGRATION_STEP: + icon, detail = self.step_status(old_dct_progression, step) + steps.append( + { + "step": step, + "icon": icon, + "label": t(label), + "detail": detail, + } + ) + lst_version = self.version_bumps(old_dct_progression) + done = old_dct_progression.get("state_4_upgrade_odoo_lst") or [] + return { + "file": os.path.basename(migration_file), + "database": old_dct_progression.get("config_database_name") or "?", + "target": old_dct_progression.get("target_odoo_version") or "?", + "started": old_dct_progression.get("date_create") or "?", + "steps": steps, + "versions": [ + { + "version": version, + "done": bool(i < len(done) and done[i]), + } + for i, version in enumerate(lst_version) + ], + } + + @staticmethod + def print_resume(ctx): + """Render the resume screen on the terminal.""" + print() + print(f"📍 {t('Migration in progress')}") + # Pad in code, not in the translations: the labels differ in length + # between languages and a hardcoded padding misaligns the colons. + print(f" {t('File'):<9}: {ctx['file']}") + print( + f" {t('Database'):<9}: {ctx['database']}" + f" · {t('Target')} : {ctx['target']}" + ) + print(f" {t('Started'):<9}: {ctx['started']}") + print() + print(f" {t('Steps')} :") + for item in ctx["steps"]: + print( + f" [{item['step']}] {item['icon']} " + f"{item['label']:<44} {item['detail']}" + ) + print() + print(f" [c] {t('Continue where it stopped')}") + print( + f" [0-4] {t('Replay from that step')}" + f" ({t('erases the progression of that step and the next ones')})" + ) + if ctx["versions"]: + versions = "/".join(str(v["version"]) for v in ctx["versions"]) + print( + f" [4.N] {t('Replay the upgrade from version N')}" + f" ({versions}) —" + f" {t('rebuilds the intermediate database')}" + ) + print(f" [n] {t('New migration, erase everything')}") + print(f" [r] {t('Keep the zip only, ask every question again')}") + print(f" [q] {t('Quit without doing anything')}") + + def apply_resume_answer(self, old_dct_progression, answer, ctx): + """Turn the answer into (progression, changed), or None to quit. + + THE decision point, shared by both interfaces: the TUI returns the + same answer strings as the prompt, so this logic is written once. + """ + answer = (answer or "").strip().lower() + lst_version = [v["version"] for v in ctx["versions"]] + + if answer in ("", "c"): + return old_dct_progression, False + if answer == "q": + return None + if answer == "n": + return {}, True + if answer == "r": + return { + "migration_file": old_dct_progression.get("migration_file"), + "date_create": old_dct_progression.get("date_create"), + }, True + if answer.startswith("4.") and lst_version: + target = answer.split(".", 1)[1].strip() + if target.isdigit() and int(target) in lst_version: + return ( + self.rewind_version_bump( + old_dct_progression, lst_version.index(int(target)) + ), + True, + ) + print(f"⚠️ {t('Unknown version')} : {target}") + return old_dct_progression, False + if answer.isdigit() and 0 <= int(answer) <= MIGRATION_STEP[-1][0]: + return ( + self.rewind_progression(old_dct_progression, int(answer)), + True, + ) + + print(f"⚠️ {t('Unknown choice, continuing where it stopped')}.") + return old_dct_progression, False + + def prompt_resume(self, old_dct_progression, use_tui=False): + """Show where the migration stands and ask what to do next. + + Returns (progression, changed), or None if the user quits. The old + menu exposed internal key names (« Reuse database without state_4 »), + which said nothing about what would happen; this shows the real state + of every step and lets the user replay from any of them. + """ + ctx = self.resume_context(old_dct_progression) + answer = None + if use_tui: + answer = self.resume_tui(ctx) + if answer is None: + self.print_resume(ctx) + answer = input(f"💬 {t('Your choice')} : ") + return self.apply_resume_answer(old_dct_progression, answer, ctx) + + @staticmethod + def ask_ui(): + """Interface of the migration: TUI, line-by-line prompts, or the + read-only statistics screen. Returns None to leave the tool. + + The preference can settle it in advance (TODO > Configuration); + « ask » asks. Same contract as the QEMU deployment — except for + « stats », which is never a stored default: it does nothing, so + landing there every time would only be in the way. + """ + try: + from script.todo import todo_prefs + + pref = todo_prefs.get("migration_ui") + except Exception: + pref = "ask" + if pref in ("tui", "cli"): + return pref + print(f"\n{t('Interface:')}") + print(f" [1] {t('TUI form')} *") + print(f" [2] {t('Classic questions (line by line)')}") + print(f" [3] {t('Migration statistics (read-only)')}") + print(f" [0] {t('Cancel')}") + print(f" {t('(change the default in TODO > Configuration)')}") + answer = input(t("Choice (0-3, default 1): ")).strip() + return {"0": None, "2": "cli", "3": "stats"}.get(answer, "tui") + + @staticmethod + def database_from_command(cmd): + """Nom de base visé par une commande, ou "" si indécelable. + + Sert à proposer le bon outil au bon moment quand une commande échoue. + On reconnaît les trois formes du dépôt : « -d », « --database + », et l'argument positionnel des scripts addons + (`update_addons_all.sh `, `install_addons*.sh `). + """ + if not cmd: + return "" + match = re.search(r"(?:^|\s)(?:-d|--database)[=\s]+([\w.-]+)", cmd) + if match: + return match.group(1) + match = re.search( + r"\./script/addons/\w+\.sh\s+([\w.-]+)", + cmd, + ) + return match.group(1) if match else "" + + def check_stale_cow_views(self, database_name): + """Lance le détecteur de copies COW en retard sur leur vue module. + + Purement consultatif : il n'écrit rien sans « --reset … --apply », que + l'on propose seulement après avoir montré le diff — réinitialiser une + copie peut effacer une personnalisation réelle.""" + script_path = os.path.join( + PATH_MIGRATION_GLOBAL, "reset_stale_cow_views.py" + ) + if not os.path.exists(script_path): + print(f"⚠️ {t('Tool not found')}: {script_path}") + return + status, _cmd = self.todo_upgrade_execute( + f"{PYTHON_BIN} ./{script_path} -d {database_name}", + wait_at_error=False, + ) + if status: + # Sortie 1 = des écarts ont été trouvés (le script les a listés). + warn = t("Read the diff first: a copy can hold a customisation.") + print(f"\n💡 {t('To reset one of them onto its module view:')}") + print( + f" {PYTHON_BIN} ./{script_path} -d {database_name}" + f" --reset --apply" + ) + print(f" {warn}") + + def show_stats(self): + """Écran de statistiques, en lecture seule : rien n'est écrit, ni + dans la base ni dans le journal de migration.""" + from script.todo import migration_stats as ms + + if not os.path.exists(UPGRADE_DATABASE_CONFIG_LOG): + print(f"\nℹ️ {t('No migration in progress to resume.')}") + return + dct = self.read_progression() + if not dct: + print(f"\nℹ️ {t('No migration in progress to resume.')}") + return + ctx = self.resume_context(dct) + database_name = dct.get("config_database_name") or "" + stats = ms.compute( + dct, + ctx, + database_name, + self.read_uninstall_module_list, + PATH_MIGRATION_PRIVATE, + PATH_MIGRATION_GLOBAL, + ) + + while True: + self.print_stats(ctx, stats) + print(f"\n [1] {t('Removed modules, with their reason')}") + print(f" [2] {t('Removed modules, comma-separated (copy)')}") + print(f" [3] {t('COW views: snapshots and differences')}") + print(f" [4] {t('Recorded decisions (journal)')}") + print(f" [5] {t('Executed commands (last 30)')}") + print(f" [0] {t('Back')}") + answer = input(f"💬 {t('Your choice')} : ").strip() + if answer in ("", "0"): + return + if answer == "1": + for version, detail in sorted(stats["uninstall"].items()): + print(f"\n── {version - 1}.0 → {version}.0 ──") + self.print_uninstall_reason(detail) + elif answer == "2": + flat = ms.flat_module_list(stats["uninstall"]) + print(f"\n{len(flat)} {t('modules')} :\n") + print(",".join(flat)) + elif answer == "3": + self.stats_cow(stats, database_name) + elif answer == "4": + for line in stats["journal"]["comments"]: + print(f" · {line}") + if not stats["journal"]["comments"]: + print(f" {t('nothing recorded')}") + elif answer == "5": + for line in stats["journal"]["commands"][-30:]: + print(f" $ {line}") + else: + print(f"⚠️ {t('Unknown choice, continuing where it stopped')}.") + + @staticmethod + def print_stats(ctx, stats): + """Rend le tableau de bord de la migration.""" + print(f"\n📊 {t('Migration statistics')}") + print(f" {t('File'):<11}: {ctx['file']}") + print( + f" {t('Database'):<11}: {ctx['database']}" + f" · {t('Target')} : {ctx['target']}" + ) + print( + f" {t('Started'):<11}: {ctx['started']}" + f" · {t('elapsed')} {stats['delay']}" + ) + + print(f"\n── {t('Level reached')} ──") + done = sum(1 for v in ctx["versions"] if v["done"]) + total = len(ctx["versions"]) or 1 + line = " " + for item in ctx["versions"]: + mark = "✅" if item["done"] else "⬜" + line += f"{item['version'] - 1}.0→{item['version']}.0 {mark} " + print(line) + print( + f" {done}/{len(ctx['versions'])} " + f"{t('version bumps migrated')} ({done * 100 // total} %)" + ) + print( + " " + + " ".join(f"[{s['step']}]{s['icon']}" for s in ctx["steps"]) + ) + + print(f"\n── {t('Modules')} ──") + if stats["origin_count"]: + print(f" {t('At the start'):<24}: {stats['origin_count']}") + for version, count, delta in stats["evolution"]: + change = "" if delta is None else f" ({delta:+d})" + print(f" {f'{version}.0':<24}: {count}{change}") + print(f" {t('Removed in total'):<24}: {stats['removed_total']}") + if stats["missing"]: + print(f" {t('Reported missing'):<24}: {len(stats['missing'])}") + if stats["duplicate"]: + print(f" {t('Duplicated'):<24}: {len(stats['duplicate'])}") + + if stats["fixes"]: + print(f"\n── {t('Migration fixes')} ──") + for fix in stats["fixes"]: + mark = "✅" if fix["applied"] else "⬜" + print(f" {mark} {fix['version']}.0 {fix['file']}") + + print(f"\n── {t('COW views')} ──") + if stats["cow"]: + for snap in stats["cow"]: + print( + f" {snap['label']:<18} {str(snap['count']):>4} " + f"{t('views')} {snap['taken_at']}" + ) + else: + print(f" {t('no snapshot')}") + + print(f"\n── {t('Journal')} ──") + print( + f" {len(stats['journal']['commands'])} {t('commands')}, " + f"{len(stats['journal']['comments'])} {t('recorded decisions')}" + ) + + def stats_cow(self, stats, database_name): + """Instantanés COW, et différence entre deux d'entre eux.""" + snaps = stats["cow"] + if len(snaps) < 2: + print(f" {t('Need two snapshots to diff.')}") + return + for index, snap in enumerate(snaps, 1): + print( + f" [{index}] {snap['label']:<18} " + f"{str(snap['count']):>4} {t('views')} {snap['taken_at']}" + ) + raw = input( + f"💬 {t('Diff which two? (e.g. 1,2 — blank to skip)')} : " + ).strip() + parts = [p.strip() for p in raw.replace(",", " ").split()] + if len(parts) != 2 or not all(p.isdigit() for p in parts): + return + first, second = (int(p) - 1 for p in parts) + if not (0 <= first < len(snaps) and 0 <= second < len(snaps)): + return + self.diff_cow_views( + database_name, snaps[first]["label"], snaps[second]["label"] + ) + + @staticmethod + def resume_tui(ctx): + """Resume screen as a TUI. Returns the SAME answer strings as the + prompt, or None when textual is missing (fall back to the prompt).""" + from script.todo import textual_setup + + if not textual_setup.ensure(): + return None + try: + from script.todo.migration_form import run_resume_tui + + return run_resume_tui(ctx) + except ImportError: + return None + + @staticmethod + def version_bumps(dct_progression): + """Odoo versions the step 4 loop walks through, e.g. [13, 14, ..., 18]. + + The per-version lists all end on the target, so the first bump is + « target - len + 1 ». Returns [] when step 4 has not started. + """ + total = max( + [ + len(value) + for key, value in dct_progression.items() + if key.startswith("state_4_") + and key.endswith("_odoo_lst") + and isinstance(value, list) + ] + or [0] + ) + if not total: + return [] + try: + last = int(float(dct_progression["target_odoo_version"])) + except (KeyError, TypeError, ValueError): + return [] + return list(range(last - total + 1, last + 1)) + + @staticmethod + def rewind_version_bump(old_dct_progression, index): + """Replay the step 4 loop from one version bump onwards. + + Only the per-version lists are trimmed from `index`: earlier bumps stay + migrated and steps 0 to 3 are untouched. Resetting the clone entry is + the point — the intermediate database of a failed bump is half + migrated, so it must be dropped and rebuilt from the previous version + rather than upgraded again. + """ + dct_kept = dict(old_dct_progression) + for key, value in old_dct_progression.items(): + if ( + key.startswith("state_4_") + and isinstance(value, list) + and key.endswith(("_odoo_lst", "_module")) + ): + dct_kept[key] = [ + item if i < index else False + for i, item in enumerate(value) + ] + return dct_kept + + @staticmethod + def rewind_progression(old_dct_progression, step): + """Drop the progression of `step` and of every later step. + + Configuration answers (config_*), the zip and the target version are + kept: they are decisions, not progress. Only « state_* » is rewound. + """ + dct_kept = {} + for key, value in old_dct_progression.items(): + if not key.startswith("state_"): + dct_kept[key] = value + continue + index = key[len("state_") :].split("_", 1)[0] + if index.isdigit() and int(index) < step: + dct_kept[key] = value + # The module search fills an in-memory dict the later steps rely on; + # it must run again even when step 0 itself is kept. + dct_kept["state_0_search_missing_module"] = False + print( + f"⏪ {t('Replaying from step')} {step} —" + f" {t(dict(MIGRATION_STEP)[step])}" + ) + return dct_kept + def on_file_selected(self, file_path): self.file_path = file_path todo_file_browser.exit_program() @@ -399,98 +921,30 @@ class TodoUpgrade: self.dct_module_per_dct_version_path = {} default_database_name = "test" + # L'écran de statistiques ne fait rien : on y revient autant de fois + # qu'on veut, et on repose ensuite le choix d'interface. + while True: + ui = self.ask_ui() + if ui is None: + return + if ui != "stats": + break + self.show_stats() + use_tui = ui == "tui" + if os.path.exists(UPGRADE_DATABASE_CONFIG_LOG): - with open(UPGRADE_DATABASE_CONFIG_LOG, "r") as f: - try: - old_dct_progression = json.load(f) - self.dct_progression = { - "migration_file": old_dct_progression.get( - "migration_file" - ), - # More useful to ask this question each time - "target_odoo_version": old_dct_progression.get( - "target_odoo_version" - ), - "date_create": old_dct_progression.get("date_create"), - } - except json.decoder.JSONDecodeError: - print( - f'⚠️ The config file "{UPGRADE_DATABASE_CONFIG_LOG}" is invalid, ignore it.' - ) - - print( - f"✨ Detected migration \"{self.dct_progression.get('migration_file')}\" " - f"to version \"{self.dct_progression.get('target_odoo_version')}\", " - f"please select an option." - ) - - print("[1] Erase progression for a new migration") - print("[2] Reuse database with new process") - print( - "[3] Reuse database without state_4, before looping on next version" - ) - print("[4] Reuse database without configuration") - erase_progression_input = ( - input("💬 Select an option or press to continue : ") - .strip() - .lower() - ) - self.dct_progression = {} - if erase_progression_input in ["2", "3", "4"]: - with open(UPGRADE_DATABASE_CONFIG_LOG, "r") as f: - try: - old_dct_progression = json.load(f) - self.dct_progression = { - "migration_file": old_dct_progression.get( - "migration_file" - ), - # More useful to ask this question each time - # "target_odoo_version": old_dct_progression.get( - # "target_odoo_version" - # ), - "date_create": old_dct_progression.get( - "date_create" - ), - } - for key, value in old_dct_progression.items(): - if erase_progression_input == "3": - if ( - key.startswith("state_0") - or key.startswith("state_1") - or key.startswith("state_2") - or key.startswith("state_3") - ): - self.dct_progression[key] = value - if key.startswith(f"config_state") and not ( - key.startswith(f"config_state_0") - or key.startswith(f"config_state_1") - or key.startswith(f"config_state_2") - or key.startswith(f"config_state_3") - ): - continue - if ( - key.startswith("config_") - and erase_progression_input != "4" - ): - self.dct_progression[key] = value - # Force to search missing module to fill dict - self.dct_progression[ - "state_0_search_missing_module" - ] = False - except json.decoder.JSONDecodeError: - print( - f"⚠️ The config file '{UPGRADE_DATABASE_CONFIG_LOG}' is invalid, ignore it." - ) - - self.write_config() - elif erase_progression_input not in ["1"]: - with open(UPGRADE_DATABASE_CONFIG_LOG, "r") as f: - try: - self.dct_progression = json.load(f) - except json.decoder.JSONDecodeError: - print( - f"⚠️ The config file '{UPGRADE_DATABASE_CONFIG_LOG}' is invalid, ignore it." - ) + old_dct_progression = self.read_progression() + if old_dct_progression: + resumed = self.prompt_resume(old_dct_progression, use_tui) + if resumed is None: + return + self.dct_progression, changed = resumed + if changed: + self.write_config() + elif use_tui: + print(f"ℹ️ {t('No migration in progress to resume.')}") + elif use_tui: + print(f"ℹ️ {t('No migration in progress to resume.')}") if "migration_file" in self.dct_progression: self.file_path = self.dct_progression["migration_file"] @@ -767,6 +1221,31 @@ class TodoUpgrade: self.write_config() print("✅ -> Restore database") + already_update_state_1 = False + + if not self.dct_progression.get("state_1_update_all"): + print( + "[1] Update all addons before neutralize (already neutralize by odoo if supported)" + ) + wait_continue = ( + input( + "💬 Do you need to upgrade before a database neutralization, press to ignore : " + ) + .strip() + .lower() + ) + if wait_continue == "1": + status, cmd_executed = self.todo_upgrade_execute( + f"./script/addons/update_addons_all.sh {database_name}", + single_source_odoo=True, + ) + + if not status: + already_update_state_1 = True + self.dct_progression["state_1_update_all"] = True + self.write_config() + + print("✅ -> Update database before neutralize by module") print("✅ -> Neutralize database") if do_neutralize: @@ -784,8 +1263,9 @@ class TodoUpgrade: # ./script/database/migrate/process_backup_file.py --path_backup_zip image_db/db.zip --path_output_zip image_db/dbFIX.zip --word_to_delete discuss_channel_channel_type_not_null # Puis faire un retry de la commande, sinon rien - self.dct_progression["state_1_neutralize_database"] = True - self.write_config() + # Only record the step when it actually succeeded. The previous + # unconditional assignment made the test below dead code: a failed + # neutralization was remembered as done and skipped on resume. if not status: self.dct_progression["state_1_neutralize_database"] = True self.write_config() @@ -801,18 +1281,12 @@ class TodoUpgrade: ) if not is_state_4_reach_open_upgrade: - lst_module_to_uninstall = [] - uninstall_module_list_file = os.path.join( - "script", - "odoo", - "migration", - f"uninstall_module_list_odoo{start_version * 10}_to_odoo{(start_version + 1) * 10}.txt", + lst_module_to_uninstall, lst_uninstall_reason = ( + self.read_uninstall_module_list(start_version, database_name) ) - if os.path.exists(uninstall_module_list_file): - with open(uninstall_module_list_file, "r") as f: - lst_module_to_uninstall = [ - a.strip() for a in f.readline().split() - ] + if lst_uninstall_reason: + print("✨ Modules to uninstall before migration :") + self.print_uninstall_reason(lst_uninstall_reason) if config_state_1_uninstall_module: lst_module_to_uninstall = ( @@ -856,7 +1330,10 @@ class TodoUpgrade: print(f"🔷 {msg}") self.add_comment_progression(msg) - if not self.dct_progression.get("state_2_update_all"): + if ( + not self.dct_progression.get("state_2_update_all") + and not already_update_state_1 + ): status, cmd_executed = self.todo_upgrade_execute( f"./script/addons/update_addons_all.sh {database_name}", single_source_odoo=True, @@ -865,6 +1342,20 @@ class TodoUpgrade: self.dct_progression["state_2_update_all"] = True self.write_config() + # Predict the website COW views that the NEXT version bump will break. + # A copy-on-write view freezes the structure of the module view it was + # copied from; when that module view changes mode between two versions, + # the copy keeps an arch written for the old mode and the upgrade dies + # on « Element ... cannot be located in parent view ». Reporting it here + # -- before hours of migration -- leaves time to arbitrate. + # Only the next bump can be predicted: the modes in database describe + # the current version. + self.todo_upgrade_execute( + f"{PYTHON_BIN} ./script/odoo/migration/check_cow_views.py" + f" -d {database_name} -t odoo{start_version + 1}.0", + wait_at_error=False, + ) + msg = "3 - Clean up database before data migration" print(f"🔷 {msg}") self.add_comment_progression(msg) @@ -1003,7 +1494,20 @@ class TodoUpgrade: # Duplicate database cmd_clone_database = f"./odoo_bin.sh db --clone --from_database {last_database_name} --database {database_name_upgrade}" - self.todo_upgrade_execute(cmd_clone_database) + status, cmd_executed = self.todo_upgrade_execute( + cmd_clone_database + ) + + # Everything downstream runs against this clone: if it failed, + # do not mark it done (a rerun would skip the clone and migrate + # a missing or truncated database). + if status: + print( + f"❌ -> Clone to Odoo{next_version} FAILED (status" + f" {status}). Stopping: '{database_name_upgrade}' is" + " not usable." + ) + return lst_clone_odoo[index] = True self.dct_progression["state_4_clone_odoo_lst"] = lst_clone_odoo @@ -1022,9 +1526,24 @@ class TodoUpgrade: ) if not lst_module_uninstall_module[index]: - lst_module_to_uninstall = config_state_4_uninstall_module[ - index - ] + lst_module_to_uninstall = ( + config_state_4_uninstall_module[index] or [] + ) + # Same file convention as step 1, one file per version bump: + # uninstall_module_list_odoo130_to_odoo140.txt is read HERE, + # right before the 13 -> 14 data migration. Without this the + # per-bump files existed in name only and were never read. + lst_file, lst_detail = self.read_uninstall_module_list( + next_version - 1, database_name + ) + if lst_detail: + print( + f"✨ Modules to uninstall before Odoo{next_version} :" + ) + self.print_uninstall_reason(lst_detail) + lst_module_to_uninstall = list( + dict.fromkeys(list(lst_module_to_uninstall) + lst_file) + ) if lst_module_to_uninstall: self.uninstall_from_database( @@ -1475,18 +1994,47 @@ class TodoUpgrade: if not lst_fix_migration_odoo[index]: print("") - file_path_fix_migration = os.path.join( - "script", - "odoo", - "migration", - f"fix_migration_odoo{(next_version-1)*10}_to_odoo{next_version*10}.py", + stem = os.path.join( + PATH_MIGRATION_GLOBAL, + f"fix_migration_odoo{(next_version - 1) * 10}" + f"_to_odoo{next_version * 10}", ) - if os.path.exists(file_path_fix_migration): - self.todo_upgrade_execute( - f"cat ./{file_path_fix_migration} | ./odoo{next_version}.0/odoo/odoo-bin shell -d {database_name_upgrade}", + # Two flavours. « .sql » runs through psql: no Odoo registry, + # so it works on a database not yet migrated -- exactly when + # loading it with the TARGET version's code would fail. « .py » + # is piped into the Odoo shell when the ORM is really needed. + file_path_fix_migration = "" + cmd_fix_migration = "" + if os.path.exists(f"{stem}.sql"): + file_path_fix_migration = f"{stem}.sql" + cmd_fix_migration = ( + f"psql -v ON_ERROR_STOP=1 -d {database_name_upgrade}" + f" -f ./{file_path_fix_migration}" + ) + elif os.path.exists(f"{stem}.py"): + file_path_fix_migration = f"{stem}.py" + cmd_fix_migration = ( + f"cat ./{file_path_fix_migration} |" + f" ./odoo{next_version}.0/odoo/odoo-bin shell" + f" -d {database_name_upgrade}" + ) + if file_path_fix_migration: + status, cmd_executed = self.todo_upgrade_execute( + cmd_fix_migration, single_source_odoo=True, ) + # A fix that did not run must not be recorded as applied, + # otherwise the rerun skips it and OpenUpgrade hits the very + # problem the fix exists to prevent. + if status: + print( + f"❌ -> Fix migration Odoo{next_version} FAILED" + f" (status {status}):" + f" {file_path_fix_migration}" + ) + return + lst_fix_migration_odoo[index] = file_path_fix_migration self.dct_progression["state_4_fix_migration_odoo_lst"] = ( lst_fix_migration_odoo @@ -1545,13 +2093,68 @@ class TodoUpgrade: cmd_upgrade = f"./run.sh --upgrade-path=./odoo{next_version}.0/OCA_OpenUpgrade/openupgrade_scripts/scripts --update all -c config.conf --stop-after-init --no-http --load=base,web,openupgrade_framework -d {database_name_upgrade}" lst_upgrade_odoo[index] = cmd_upgrade - self.todo_upgrade_execute( + # Record the website COW views before the data migration. The + # upgrade silently deletes and recreates copies (measured on + # 12->13: 16 copies dropped, 13 created, children re-parented), + # and rewrites the arch of many others. Without a before/after + # record, "the site looks wrong" cannot be investigated. + self.snapshot_cow_views( + database_name_upgrade, f"before_{next_version}" + ) + + # Take the copies that cannot survive this bump out of the way, + # otherwise the data migration dies on them. Offered, not + # forced: their arch is a real customization. + self.neutralize_cow_views(database_name_upgrade, next_version) + + # wait_at_error=False on purpose: the generic « [1] to redo the + # command » would replay OpenUpgrade on a database it has just + # half migrated, which never recovers. The failure is handled + # below by dropping the clone flag so the replay REBUILDS the + # intermediate database from the previous version. + status, cmd_executed = self.todo_upgrade_execute( cmd_upgrade, new_env={ "OPENUPGRADE_TARGET_VERSION": f"{next_version}.0" }, + wait_at_error=False, + ) + + # This is THE data migration. Recording it as done when it + # failed used to send the loop to the next version on top of a + # half-migrated database. Stop here instead: the state stays + # unset, so a rerun replays this version. + if status: + # The intermediate database is now half migrated and must + # not be reused: drop the clone flag so the rerun rebuilds + # it from the pristine source. Without this the replay + # would restart OpenUpgrade on top of the broken clone. + lst_clone_odoo[index] = False + self.dct_progression["state_4_clone_odoo_lst"] = ( + lst_clone_odoo + ) + self.write_config() + print( + f"\n❌ -> Database migration to Odoo{next_version}" + f" FAILED (status {status}).\n" + f" '{database_name_upgrade}' is now half migrated:" + " replaying the command on it would never recover, so" + " it is NOT offered.\n" + " The clone step has been reset. Fix the cause, then" + " relaunch the migration and answer [c] (continue):" + f" '{database_name_upgrade}' will be dropped and" + " rebuilt from the previous version before retrying." + ) + return + + self.snapshot_cow_views( + database_name_upgrade, f"after_{next_version}" + ) + self.diff_cow_views( + database_name_upgrade, + f"before_{next_version}", + f"after_{next_version}", ) - # TODO detect error self.dct_progression["state_4_upgrade_odoo_lst"] = ( lst_upgrade_odoo @@ -1738,18 +2341,217 @@ class TodoUpgrade: } return dct_module + def snapshot_cow_views(self, database_name, label): + """Record the website COW views of a database under private/. + + Never blocks the migration: a snapshot is forensic material, its + absence must not stop an upgrade. + """ + self.todo_upgrade_execute( + f"{PYTHON_BIN} ./script/odoo/migration/snapshot_cow_views.py" + f" -d {database_name} -l {label}", + wait_at_error=False, + ) + + def neutralize_cow_views(self, database_name, next_version): + """Offer to neutralize the COW views that would break this bump. + + Renaming their key unpairs them from the module view, so the upgrade + stops choking on them. Nothing is deleted and the operation is + reversible (neutralize_cow_views.py --restore), but the choice belongs + to the user: those copies carry real customizations. + """ + cmd = ( + f"{PYTHON_BIN} ./script/odoo/migration/neutralize_cow_views.py" + f" -d {database_name} -t odoo{next_version}.0" + ) + status, cmd_executed, output = self.todo_upgrade_execute( + cmd, get_output=True, wait_at_error=False + ) + if "No website COW view to neutralize" in "\n".join(output or []): + return + + answer = ( + input( + "💬 Neutralize these copies so the upgrade can proceed?" + " Their arch is kept and the change is reversible." + " (Y/n) : " + ) + .strip() + .lower() + ) + if answer == "n": + print( + "⚠️ -> Skipped. The data migration will very likely stop on" + " these views." + ) + return + self.todo_upgrade_execute(f"{cmd} --apply", wait_at_error=False) + + def diff_cow_views(self, database_name, label_before, label_after): + """Print what the version bump did to the website COW views.""" + directory = os.path.join( + PATH_MIGRATION_PRIVATE, database_name, "cow_snapshots" + ) + path_before = os.path.join(directory, f"{label_before}.json") + path_after = os.path.join(directory, f"{label_after}.json") + if not (os.path.exists(path_before) and os.path.exists(path_after)): + return + self.todo_upgrade_execute( + f"{PYTHON_BIN} ./script/odoo/migration/snapshot_cow_views.py" + f" --diff {path_before} {path_after}", + wait_at_error=False, + ) + + @staticmethod + def parse_module_list_file(file_path): + """Read a module list file, return [(module, reason), ...]. + + Accepted syntax, one module per line with an optional justification: + + queue_job # blocks 12->13, trigger queue_job_notify + mgmtsystem_hazard # not ported to 13.0 + + Commas and several names per line are also accepted, so a list copied + from a command line works as-is. Blank lines and full-line comments are + ignored. + + The previous parser was « f.readline().split() »: it kept only the FIRST + line, so a multi-line list was silently truncated, and a comma-separated + list collapsed into one bogus module name. + """ + lst_module = [] + with open(file_path, "r", encoding="utf-8") as f: + for line in f: + content, _, reason = line.partition("#") + for module_name in content.replace(",", " ").split(): + lst_module.append((module_name, reason.strip())) + return lst_module + + @staticmethod + def print_uninstall_reason(lst_detail): + """Show WHY each module goes away. + + Which modules must be dropped depends on the database, and a removal + without a stated reason is a decision nobody can review later. + """ + for name, reason, origin in lst_detail: + print( + f" - {name}" + + (f" — {reason}" if reason else " — ⚠️ no reason given") + + f" [{origin}]" + ) + + def read_uninstall_module_list(self, start_version, database_name): + """Modules to uninstall before migrating start_version -> next. + + Reads the per-database private list first, then the shared versioned + defaults; duplicates are dropped, keeping the first occurrence. + + Returns (lst_module, lst_detail) where lst_detail carries + (module, reason, origin_file) so the caller can justify each removal. + """ + file_name = ( + f"uninstall_module_list_odoo{start_version * 10}" + f"_to_odoo{(start_version + 1) * 10}.txt" + ) + lst_path = [ + os.path.join(PATH_MIGRATION_PRIVATE, database_name, file_name), + os.path.join(PATH_MIGRATION_GLOBAL, file_name), + ] + + lst_module = [] + lst_detail = [] + for file_path in lst_path: + if not os.path.exists(file_path): + continue + for module_name, reason in self.parse_module_list_file(file_path): + if module_name in lst_module: + continue + lst_module.append(module_name) + lst_detail.append((module_name, reason, file_path)) + return lst_module, lst_detail + + def split_present_missing(self, lst_module): + """Split a module list into (present, missing) against the ACTIVE code. + + « Missing » means the addons path no longer holds the module — not + that it is absent from the database. The distinction matters because + check_addons_exist.py refuses the WHOLE uninstall for a single missing + name, so the modules that ARE there never get uninstalled either. + """ + lst_missing, _lst_duplicate = self.check_addons_exist(lst_module) + set_missing = set(lst_missing or []) + return ( + [name for name in lst_module if name not in set_missing], + [name for name in lst_module if name in set_missing], + ) + + def prompt_uninstall_missing(self, lst_present, lst_missing): + """Ask what to do when part of the list has no code left. + Returns the list to actually uninstall (possibly empty).""" + print() + print( + f"⚠️ {len(lst_missing)} " + f"{t('modules of the list have no code in the active Odoo:')}" + ) + for name in lst_missing: + print(f" {name}") + print(f" {t('Odoo cannot uninstall a module whose code is gone;')}") + print(f" {t('one of them fails the whole uninstall command.')}") + print() + if lst_present: + print( + f" {len(lst_present)} " + f"{t('are present and can be uninstalled:')}" + ) + for name in lst_present: + print(f" {name}") + else: + print(f" {t('No module of the list is present.')}") + print() + if lst_present: + print( + f" [1] {t('Uninstall the present ones, skip the missing')}" + " *" + ) + print(f" [2] {t('Try the whole list anyway (it will fail)')}") + print(f" [3] {t('Uninstall nothing, continue')}") + answer = input(f"💬 {t('Your choice')} : ").strip() + if answer == "2": + return lst_present + lst_missing + if answer == "3" or not lst_present: + return [] + return lst_present + def uninstall_from_database( self, lst_module_to_uninstall, database_name, actual_version ): if not lst_module_to_uninstall: return + # Sort out what the active code still holds BEFORE calling the script: + # it aborts on the first missing name and takes the rest down with it. + lst_present, lst_missing = self.split_present_missing( + lst_module_to_uninstall + ) + if lst_missing: + self.add_comment_progression( + "uninstall - no code for: " + ", ".join(lst_missing) + ) + lst_module_to_uninstall = self.prompt_uninstall_missing( + lst_present, lst_missing + ) + if not lst_module_to_uninstall: + print(f"⏭ {t('Nothing uninstalled.')}") + return uninstall_module = ",".join(lst_module_to_uninstall) self.todo_upgrade_execute( f"./script/addons/uninstall_addons.sh {database_name} {uninstall_module}", single_source_odoo=True, ) - # Update list installed module + # Update list installed module — only what was REALLY uninstalled, so + # a module left in place stays counted as installed. self.dct_module_per_version[actual_version] = sorted( list( set(self.dct_module_per_version[actual_version]) @@ -1894,18 +2696,38 @@ class TodoUpgrade: self.lst_command_executed.append(cmd_executed) self.dct_progression["command_executed"] = self.lst_command_executed self.write_config() - if status and wait_at_error: - print("[1] to redo the command") - wait_status = ( - input( - "💬 Error detected, press to continue or ctrl+c to stop : " + # None means « the command never reported a status » -> treat it as a + # failure, never as a success (defence in depth: exec_command_live now + # always sets one, but a silent None must not skip this prompt). + if (status is None or status) and wait_at_error: + database_name = self.database_from_command(cmd) + while True: + print("[1] to redo the command") + if database_name: + print( + f"[2] {t('Check the COW views that drifted')}" + f" ({database_name})" + ) + wait_status = ( + input( + "💬 Error detected, press to continue or ctrl+c to" + " stop : " + ) + .strip() + .lower() ) - .strip() - .lower() - ) - # psycopg2.errors.UndefinedTable: relation "discuss_channel" does not exist - # LIGNE 1 : SELECT "discuss_channel"."id" FROM "discuss_channel" WHERE (... + # psycopg2.errors.UndefinedTable: relation "discuss_channel" does not exist + # LIGNE 1 : SELECT "discuss_channel"."id" FROM "discuss_channel" WHERE (... + + if wait_status == "2" and database_name: + # Le motif d'échec le plus fréquent ici est « Element + # cannot be located in parent view » : une copie + # COW en retard sur sa vue module. On propose l'outil sur + # place, puis on repose le choix pour rejouer. + self.check_stale_cow_views(database_name) + continue + break if wait_status == "1": return self.todo_upgrade_execute(