From b28b734338dbe8575017e9497ac3640c36daf924 Mon Sep 17 00:00:00 2001 From: Mathieu Benoit Date: Mon, 10 Aug 2026 03:10:50 -0400 Subject: [PATCH] [ADD] migration: see and repair the website COW views MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A copy-on-write view freezes the module view it came from; the module moves on and the upgrade dies hours later on a missing anchor. These tools predict, snapshot, diff, neutralize and reset them. The migration screen gains the real state of each step, replay from any of them, and statistics. --- FR --- Une vue copy-on-write fige la vue de module dont elle vient ; le module évolue et la mise à niveau meurt des heures plus tard sur un point d'ancrage absent. Ces outils les prévoient, photographient, comparent, neutralisent et réinitialisent. L'écran de migration gagne l'état réel de chaque étape, la reprise depuis n'importe laquelle, et des statistiques. Assisted-by: Claude Opus 5 --- private/.gitignore | 5 + script/execute/execute.py | 17 +- script/odoo/migration/README.base.md | 41 + script/odoo/migration/README.fr.md | 25 +- script/odoo/migration/README.md | 23 + script/odoo/migration/check_cow_views.py | 347 ++++++ .../fix_migration_odoo130_to_odoo140.sql | 39 + script/odoo/migration/neutralize_cow_views.py | 158 +++ .../odoo/migration/reset_stale_cow_views.py | 365 ++++++ script/odoo/migration/snapshot_cow_views.py | 213 ++++ script/todo/migration_form.py | 186 +++ script/todo/migration_stats.py | 201 +++ script/todo/todo_i18n.py | 293 +++++ script/todo/todo_upgrade.py | 1084 +++++++++++++++-- 14 files changed, 2864 insertions(+), 133 deletions(-) create mode 100755 script/odoo/migration/check_cow_views.py create mode 100644 script/odoo/migration/fix_migration_odoo130_to_odoo140.sql create mode 100755 script/odoo/migration/neutralize_cow_views.py create mode 100755 script/odoo/migration/reset_stale_cow_views.py create mode 100755 script/odoo/migration/snapshot_cow_views.py create mode 100644 script/todo/migration_form.py create mode 100644 script/todo/migration_stats.py diff --git a/private/.gitignore b/private/.gitignore index 5e440ed..12cb007 100644 --- a/private/.gitignore +++ b/private/.gitignore @@ -1 +1,6 @@ *.kdbx + +# Per-database migration lists (modules to uninstall/install before a version +# bump). They describe one specific database, never a shared default, so they +# must not be versioned. Shared defaults belong to script/odoo/migration/. +odoo/ diff --git a/script/execute/execute.py b/script/execute/execute.py index 5ef684a..cfb9ff3 100644 --- a/script/execute/execute.py +++ b/script/execute/execute.py @@ -146,7 +146,16 @@ class Execute: "You cannot execute Odoo command if no version is" f" installed. Command : {redact_secrets(command)}" ) - return -1 + # Return the SAME shape the caller asked for. A bare int here + # made callers doing « status, cmd = exec_command_live(...) » + # crash with ValueError instead of seeing the failure. + if return_status_and_output_and_command: + return 1, command, [] + if return_status_and_command: + return 1, command + if return_status_and_output: + return 1, [] + return 1 command = f"source ./.venv.{source_odoo}/bin/activate && {command}" if new_window and self.cmd_source_default: command = self.cmd_source_default % command @@ -191,10 +200,16 @@ class Execute: if process.returncode != 0 and not quiet: print("Command returned error code:" f" {process.returncode}") + # An exception MUST report a failure. exit_code stays None otherwise, + # and None is falsy: callers testing « if not status: » would mark the + # step as done, and « if status and wait_at_error » would skip the error + # prompt. A crashed command was therefore recorded as a success. except FileNotFoundError: + exit_code = 1 if not quiet: print(f"Error: Command '{redact_secrets(command)}' not found.") except Exception as e: + exit_code = 1 if not quiet: print(f"An error occurred: {redact_secrets(str(e))}") process_end_time = time.time() diff --git a/script/odoo/migration/README.base.md b/script/odoo/migration/README.base.md index 7856f03..a29e31f 100644 --- a/script/odoo/migration/README.base.md +++ b/script/odoo/migration/README.base.md @@ -21,5 +21,46 @@ source ./.venv.odoo15.0_python3.8.20/bin/activate && cat ./script/odoo/migration Check [uninstall_module_list_odoo140_to_odoo150.txt](uninstall_module_list_odoo140_to_odoo150.txt) +## Module lists to uninstall + +Before a version bump, the migration uninstalls the modules listed in +`uninstall_module_list_odoo_to_odoo.txt`. Two locations are read, the +private one first, and the results are merged (duplicates dropped): + +1. `private/odoo/migration//uninstall_module_list_odooXX0_to_odooYY0.txt` + — specific to ONE database, not versioned. Which modules must be dropped + depends on the data, so this is where nearly every entry belongs. +2. `script/odoo/migration/uninstall_module_list_odooXX0_to_odooYY0.txt` + — shared defaults, versioned, valid for every database. + +Syntax: one module per line, with a justification after `#`. Commas and several +names per line are accepted; blank lines and full-line comments are ignored. +A module without a stated reason is flagged at runtime: removing a module is a +decision someone must be able to review later. + Consultez [uninstall_module_list_odoo140_to_odoo150.txt](uninstall_module_list_odoo140_to_odoo150.txt) + +## Listes de modules à désinstaller + +Avant une montée de version, la migration désinstalle les modules listés dans +`uninstall_module_list_odoo_to_odoo.txt`. Deux emplacements sont +lus, le privé d'abord, puis fusionnés (doublons éliminés) : + +1. `private/odoo/migration//uninstall_module_list_odooXX0_to_odooYY0.txt` + — propre à UNE base de données, non versionné. Les modules à supprimer + dépendent des données : c'est ici que va la quasi-totalité des entrées. +2. `script/odoo/migration/uninstall_module_list_odooXX0_to_odooYY0.txt` + — valeurs par défaut partagées, versionnées, valables pour toute base. + +Syntaxe : un module par ligne, avec une justification après `#`. Les virgules et +plusieurs noms par ligne sont acceptés ; lignes vides et commentaires pleine +ligne sont ignorés. Un module sans raison est signalé à l'exécution : supprimer +un module est une décision qui doit pouvoir être relue plus tard. + + +``` +queue_job # blocks 12->13, trigger queue_job_notify +mgmtsystem_hazard # not ported to 13.0 +web_syncer # dropped upstream +``` diff --git a/script/odoo/migration/README.fr.md b/script/odoo/migration/README.fr.md index 19ca316..35961eb 100644 --- a/script/odoo/migration/README.fr.md +++ b/script/odoo/migration/README.fr.md @@ -7,4 +7,27 @@ Exécutez ce script lors de la migration de base de données. Exemple : source ./.venv.odoo15.0_python3.8.20/bin/activate && cat ./script/odoo/migration/fix_migration_odoo140_to_odoo150.py | ./odoo15.0/odoo/odoo-bin shell -d DATABASE ``` -Consultez [uninstall_module_list_odoo140_to_odoo150.txt](uninstall_module_list_odoo140_to_odoo150.txt) \ No newline at end of file +Consultez [uninstall_module_list_odoo140_to_odoo150.txt](uninstall_module_list_odoo140_to_odoo150.txt) + +## Listes de modules à désinstaller + +Avant une montée de version, la migration désinstalle les modules listés dans +`uninstall_module_list_odoo_to_odoo.txt`. Deux emplacements sont +lus, le privé d'abord, puis fusionnés (doublons éliminés) : + +1. `private/odoo/migration//uninstall_module_list_odooXX0_to_odooYY0.txt` + — propre à UNE base de données, non versionné. Les modules à supprimer + dépendent des données : c'est ici que va la quasi-totalité des entrées. +2. `script/odoo/migration/uninstall_module_list_odooXX0_to_odooYY0.txt` + — valeurs par défaut partagées, versionnées, valables pour toute base. + +Syntaxe : un module par ligne, avec une justification après `#`. Les virgules et +plusieurs noms par ligne sont acceptés ; lignes vides et commentaires pleine +ligne sont ignorés. Un module sans raison est signalé à l'exécution : supprimer +un module est une décision qui doit pouvoir être relue plus tard. + +``` +queue_job # blocks 12->13, trigger queue_job_notify +mgmtsystem_hazard # not ported to 13.0 +web_syncer # dropped upstream +``` \ No newline at end of file diff --git a/script/odoo/migration/README.md b/script/odoo/migration/README.md index 62bf9c7..9002c6c 100644 --- a/script/odoo/migration/README.md +++ b/script/odoo/migration/README.md @@ -8,3 +8,26 @@ source ./.venv.odoo15.0_python3.8.20/bin/activate && cat ./script/odoo/migration ``` Check [uninstall_module_list_odoo140_to_odoo150.txt](uninstall_module_list_odoo140_to_odoo150.txt) + +## Module lists to uninstall + +Before a version bump, the migration uninstalls the modules listed in +`uninstall_module_list_odoo_to_odoo.txt`. Two locations are read, the +private one first, and the results are merged (duplicates dropped): + +1. `private/odoo/migration//uninstall_module_list_odooXX0_to_odooYY0.txt` + — specific to ONE database, not versioned. Which modules must be dropped + depends on the data, so this is where nearly every entry belongs. +2. `script/odoo/migration/uninstall_module_list_odooXX0_to_odooYY0.txt` + — shared defaults, versioned, valid for every database. + +Syntax: one module per line, with a justification after `#`. Commas and several +names per line are accepted; blank lines and full-line comments are ignored. +A module without a stated reason is flagged at runtime: removing a module is a +decision someone must be able to review later. + +``` +queue_job # blocks 12->13, trigger queue_job_notify +mgmtsystem_hazard # not ported to 13.0 +web_syncer # dropped upstream +``` \ No newline at end of file diff --git a/script/odoo/migration/check_cow_views.py b/script/odoo/migration/check_cow_views.py new file mode 100755 index 0000000..07321a0 --- /dev/null +++ b/script/odoo/migration/check_cow_views.py @@ -0,0 +1,347 @@ +#!/usr/bin/env python3 +# © 2021-2026 TechnoLibre (http://www.technolibre.ca) +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl) + +"""Predict which website COW views will break on the next version bump. + +Background +---------- +When a website view is customized, Odoo makes a copy-on-write (COW) copy tied +to a website_id. That copy freezes the arch AND the structure of the module +view it was copied from. + +A module view can change structure between two Odoo versions. Example measured +on a real 12.0 database: ``portal.frontend_layout`` is declared ``primary`` in +12.0 (a full QWeb template) and becomes an ``extension`` in 13.0 +(``inherit_id="web.frontend_layout"`` + xpath). During the upgrade the COW copy +follows the module and becomes an extension, but keeps its 12.0 full-template +arch. Odoo then applies the ```` root as an inheritance spec, +cannot find it in the parent, and the whole upgrade stops on:: + + ValueError: Element '' + cannot be located in parent view + +So the rule is: + + a COW view breaks when the shape its arch must have changes between + version N and version N+1. + +The discriminant is NOT ``mode``. What decides the required shape is whether +the target declares an ``inherit_id``: if it does, the arch must be inheritance +specs (````, ````, ``position=``); if it does not, the arch must be +a standalone template. Comparing ``mode`` alone misses a real case: a view +moving from a root template to ``inherit_id`` + ``primary="True"`` keeps +``mode='primary'`` on both sides yet still has to change shape, and the copy +still breaks. + +That is predictable *before* starting a multi-hour migration: the stored arch is +in the database, and the required shape is declared in the target version +sources. This script compares the two and reports the views at risk. + +It only reads: no database write, no source modification. +""" + +import argparse +import glob +import json +import os +import re +import subprocess +import sys +import xml.etree.ElementTree as ET + +# A view whose module counterpart cannot be found at all. +MODE_UNKNOWN = "unknown" + +# arch_db is text up to 15.0 and jsonb from 16.0 ({"en_US": "..."}). +RE_XML_DECLARATION = re.compile(r"<\?xml.*?\?>", re.DOTALL) +RE_FIRST_TAG = re.compile(r"<\s*([A-Za-z_][\w.:-]*)") +# Tags that carry inheritance specs rather than a standalone template. +SPEC_ROOT_TAG = ("data", "xpath") + + +def query_cow_views(database): + """Return [(id, key, mode, website_id, arch)] for every website COW view.""" + sql = ( + "SELECT id, COALESCE(key, ''), mode, website_id," + " replace(left(COALESCE(arch_db::text, ''), 400), chr(10), ' ')" + " FROM ir_ui_view WHERE website_id IS NOT NULL ORDER BY id;" + ) + result = subprocess.run( + ["psql", "-d", database, "-tAF", "|", "-c", sql], + capture_output=True, + text=True, + ) + if result.returncode: + raise RuntimeError( + f"Cannot read views from '{database}': {result.stderr.strip()}" + ) + lst_view = [] + for line in result.stdout.splitlines(): + if not line.strip(): + continue + view_id, key, mode, website_id, arch = line.split("|", 4) + lst_view.append((int(view_id), key, mode, website_id, arch)) + return lst_view + + +def arch_is_inheritance_spec(arch): + """True when the arch holds inheritance specs, not a standalone template. + + This is the real discriminant, not ``mode``. A view declared with an + ``inherit_id`` must hold specs (````, ````, or an element with + a ``position``); a root view holds a full template (````, + ``
``, ...). A copy that keeps the wrong form for what the target + version expects is exactly what raises « cannot be located in parent view ». + """ + if not arch: + return None + # From 16.0 arch_db is jsonb: take any translation, the structure is shared. + if arch.lstrip().startswith("{"): + try: + translations = json.loads(arch) + arch = next(iter(translations.values()), "") + except (ValueError, StopIteration): + # Truncated jsonb: fall through and look at the raw text. + pass + arch = RE_XML_DECLARATION.sub("", arch or "") + match = RE_FIRST_TAG.search(arch) + if not match: + return None + if match.group(1).lower() in SPEC_ROOT_TAG: + return True + # style specs. + return "position=" in arch[: match.end() + 200] + + +def load_renamed_modules(odoo_version): + """Return {old_module: new_module} from the target OpenUpgrade apriori.py. + + Without this a module renamed upstream looks absent, and every view of that + module is misreported as « module gone » instead of being checked. + """ + pattern = os.path.join(odoo_version, "**", "apriori.py") + for file_path in sorted(glob.glob(pattern, recursive=True)): + data_vars = {} + try: + with open(file_path, "r", encoding="utf-8") as f: + exec(f.read(), data_vars) # noqa: S102 - upstream data file + except Exception: + # A broken or exotic apriori.py must not stop the whole report. + continue + renamed = data_vars.get("renamed_modules") + if isinstance(renamed, dict) and renamed: + return renamed + return {} + + +def find_module_dir(odoo_version, module_name): + """Locate a module directory inside an odoo tree.""" + lst_pattern = [ + os.path.join(odoo_version, "odoo", "addons", module_name), + os.path.join(odoo_version, "odoo", "odoo", "addons", module_name), + os.path.join(odoo_version, "addons", "*", module_name), + ] + for pattern in lst_pattern: + for path in sorted(glob.glob(pattern)): + if os.path.isdir(path): + return path + return None + + +def declared_view_shape(module_dir, template_id): + """Return (mode, inherits) for a view declared in the sources, else None. + + ``inherits`` is what really matters: a declared inherit_id means the arch + must be inheritance specs. ``mode`` is kept because it is still worth + reporting, but it is NOT a reliable discriminant: a view moving from a root + template to « inherit_id + primary="True" » keeps mode='primary' on both + sides while its arch shape has to change. + """ + pattern = os.path.join(module_dir, "**", "*.xml") + for file_path in sorted(glob.glob(pattern, recursive=True)): + try: + root = ET.parse(file_path).getroot() + except ET.ParseError: + continue + for element in root.iter(): + if element.get("id") != template_id: + continue + if element.tag == "template": + inherits = bool(element.get("inherit_id")) + if str(element.get("primary", "")).lower() in ("true", "1"): + return "primary", inherits + return ("extension" if inherits else "primary"), inherits + if ( + element.tag == "record" + and element.get("model") == "ir.ui.view" + ): + mode = None + inherits = False + for field in element.findall("field"): + if field.get("name") == "mode": + mode = (field.text or "").strip() + elif field.get("name") == "inherit_id": + inherits = True + if mode: + return mode, inherits + return ("extension" if inherits else "primary"), inherits + return None + + +def analyse(database, target_version): + """Sort COW views into three buckets by comparing with the target sources. + + - at_risk : the module view changes mode -> the copy will break + - module_absent : the module itself is gone in the target version + - no_counterpart : no module view with that id, so it is a page or a record + created from the editor. Normal, and not at risk. + """ + lst_at_risk = [] + lst_module_absent = [] + lst_no_counterpart = [] + cache_shape = {} + renamed_modules = load_renamed_modules(target_version) + + for view_id, key, mode, website_id, arch in query_cow_views(database): + if not key or "." not in key: + continue + if key not in cache_shape: + module_name, _, template_id = key.partition(".") + module_dir = find_module_dir(target_version, module_name) + if module_dir is None and module_name in renamed_modules: + module_dir = find_module_dir( + target_version, renamed_modules[module_name] + ) + if module_dir is None: + cache_shape[key] = MODE_UNKNOWN + else: + cache_shape[key] = declared_view_shape(module_dir, template_id) + shape = cache_shape[key] + + if shape == MODE_UNKNOWN: + lst_module_absent.append((view_id, key, mode, website_id)) + continue + if shape is None: + lst_no_counterpart.append((view_id, key, mode, website_id)) + continue + + target_mode, target_inherits = shape + is_spec = arch_is_inheritance_spec(arch) + + # The decisive test: the target expects inheritance specs but the copy + # holds a standalone template, or the reverse. + if is_spec is not None and target_inherits != is_spec: + reason = ( + "target inherits, copy holds a standalone template" + if target_inherits + else "target is a root view, copy holds inheritance specs" + ) + lst_at_risk.append( + (view_id, key, mode, target_mode, website_id, reason) + ) + elif target_mode != mode: + # Shape is fine but the mode moves: worth reporting, less severe. + lst_at_risk.append( + ( + view_id, + key, + mode, + target_mode, + website_id, + "mode changes, arch shape unchanged", + ) + ) + return lst_at_risk, lst_module_absent, lst_no_counterpart + + +def main(): + parser = argparse.ArgumentParser( + description=( + "Report website COW views that will break on the next Odoo" + " version bump (read-only)." + ) + ) + parser.add_argument( + "-d", "--database", required=True, help="database to inspect" + ) + parser.add_argument( + "-t", + "--target_version", + required=True, + help="target Odoo source directory, e.g. odoo13.0", + ) + parser.add_argument( + "-v", + "--verbose", + action="store_true", + help="also list the editor-made pages, which are not at risk", + ) + config = parser.parse_args() + + if not os.path.isdir(config.target_version): + print( + f"❌ Target version directory '{config.target_version}' not found." + ) + return 1 + + lst_at_risk, lst_module_absent, lst_no_counterpart = analyse( + config.database, config.target_version + ) + + if not lst_at_risk: + print( + "✅ -> No website COW view changes mode in" + f" {config.target_version}." + ) + else: + print( + f"⚠️ {len(lst_at_risk)} website COW view(s) will break when moving" + f" to {config.target_version}: the copy keeps an arch whose shape" + " no longer matches what the target module view expects." + ) + for ( + view_id, + key, + mode, + target_mode, + website_id, + reason, + ) in lst_at_risk: + print( + f" - id={view_id} website={website_id} {key}" + f" : {mode} -> {target_mode} ({reason})" + ) + print( + " Arbitrate BEFORE launching the migration. To neutralize a copy," + " rename its key (UPDATE ir_ui_view SET key='zz_cow_archive.'||key," + " active=false): an unmatched key is never paired with the module" + " view, so the copy never receives the new inherit_id. Setting" + " active=false alone is NOT enough -- an inactive copy that keeps" + " the same key still shadows the module view." + ) + + if lst_module_absent: + print( + f"ℹ {len(lst_module_absent)} COW view(s) belong to a module absent" + f" from {config.target_version}:" + ) + for view_id, key, mode, website_id in lst_module_absent: + print(f" - id={view_id} website={website_id} {key} ({mode})") + + if lst_no_counterpart: + print( + f"ℹ {len(lst_no_counterpart)} COW view(s) are pages or records made" + " from the website editor (no module view of that name): not at" + " risk." + ("" if config.verbose else " Use -v to list them.") + ) + if config.verbose: + for view_id, key, mode, website_id in lst_no_counterpart: + print(f" - id={view_id} website={website_id} {key} ({mode})") + + # Informative only: never fail the migration on a warning. + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/script/odoo/migration/fix_migration_odoo130_to_odoo140.sql b/script/odoo/migration/fix_migration_odoo130_to_odoo140.sql new file mode 100644 index 0000000..cde9856 --- /dev/null +++ b/script/odoo/migration/fix_migration_odoo130_to_odoo140.sql @@ -0,0 +1,39 @@ +-- © 2021-2026 TechnoLibre (http://www.technolibre.ca) +-- License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl) +-- +-- Odoo 13.0 -> 14.0 : hand the « group_fiscal_year » security group over to +-- the module that owns it in 14.0. +-- +-- In 13.0 the group « Allow to define fiscal years of more or less than a +-- year » is declared by the core « account » module, so the database holds it +-- as account.group_fiscal_year. In 14.0 core account no longer declares it and +-- om_account_accountant (odoomates) does. During the upgrade that module finds +-- no XML id of its own, tries to CREATE the group, and hits: +-- +-- duplicate key value violates unique constraint "res_groups_name_uniq" +-- Key (category_id, name)=(9, Allow to define fiscal years ...) already exists +-- +-- Renaming the XML id makes Odoo UPDATE the existing row instead of creating a +-- duplicate. The record id is untouched, so any user assignment, access right +-- or record rule pointing at the group survives. +-- +-- Runs through psql, not the Odoo shell: at this point the database is still +-- 13.0 and loading it with the 14.0 registry is exactly what fails. + +UPDATE ir_model_data + SET module = 'om_account_accountant' + WHERE model = 'res.groups' + AND module = 'account' + AND name = 'group_fiscal_year' + -- Only when that module is actually part of this database. + AND EXISTS ( + SELECT 1 FROM ir_module_module + WHERE name = 'om_account_accountant' + AND state IN ('installed', 'to upgrade', 'to install') + ) + -- Idempotent: do nothing if the target XML id already exists. + AND NOT EXISTS ( + SELECT 1 FROM ir_model_data + WHERE module = 'om_account_accountant' + AND name = 'group_fiscal_year' + ); diff --git a/script/odoo/migration/neutralize_cow_views.py b/script/odoo/migration/neutralize_cow_views.py new file mode 100755 index 0000000..7e32425 --- /dev/null +++ b/script/odoo/migration/neutralize_cow_views.py @@ -0,0 +1,158 @@ +#!/usr/bin/env python3 +# © 2021-2026 TechnoLibre (http://www.technolibre.ca) +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl) + +"""Neutralize the website COW views that would break a version bump. + +How it works +------------ +``key`` is the only thing that pairs a website copy with the module view it +came from. A copy whose key matches nothing is never paired, so it never +receives the new ``inherit_id``, never changes shape, and never takes part in +any view combination. Renaming the key is therefore enough to take a copy out +of the way:: + + UPDATE ir_ui_view SET key = '.' || key, active = false WHERE id = ? + +Setting ``active = false`` alone would NOT work: an inactive copy that keeps +the same key still shadows the module view. + +Nothing is deleted, so ``inherit_id ondelete='restrict'`` and the +``website_page`` foreign keys are never touched, and the 12.0 arch stays in +database as a readable archive. ``--restore`` puts everything back. + +Plain psql on purpose: this must run on a database that has not been migrated +yet, where starting an Odoo shell of the target version is not guaranteed. +""" + +import argparse +import os +import subprocess +import sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +from check_cow_views import analyse # noqa: E402 + +DEFAULT_PREFIX = "zz_cow_archive" + + +def run_psql(database, sql): + """Run a statement and return stdout, raising on failure.""" + result = subprocess.run( + ["psql", "-d", database, "-tAc", sql], + capture_output=True, + text=True, + ) + if result.returncode: + raise RuntimeError( + f"Query failed on '{database}': {result.stderr.strip()}" + ) + return result.stdout.strip() + + +def neutralize(database, lst_view_id, prefix): + """Rename the key of the given views and deactivate them.""" + if not lst_view_id: + return 0 + ids = ",".join(str(view_id) for view_id in lst_view_id) + output = run_psql( + database, + "WITH updated AS (" + f" UPDATE ir_ui_view SET key = '{prefix}.' || key, active = false" + f" WHERE id IN ({ids}) AND key NOT LIKE '{prefix}.%'" + " RETURNING 1) SELECT count(*) FROM updated;", + ) + return int(output or 0) + + +def restore(database, prefix): + """Undo a neutralization: strip the prefix and reactivate.""" + output = run_psql( + database, + "WITH updated AS (" + f" UPDATE ir_ui_view SET key = substring(key from {len(prefix) + 2})," + " active = true" + f" WHERE key LIKE '{prefix}.%'" + " RETURNING 1) SELECT count(*) FROM updated;", + ) + return int(output or 0) + + +def main(): + parser = argparse.ArgumentParser( + description=( + "Neutralize the website COW views that would break a version" + " bump, by renaming their key. Dry-run unless --apply." + ) + ) + parser.add_argument("-d", "--database", required=True) + parser.add_argument( + "-t", + "--target_version", + help="target Odoo source directory, e.g. odoo13.0", + ) + parser.add_argument( + "--prefix", + default=DEFAULT_PREFIX, + help=f"archive prefix for the key (default: {DEFAULT_PREFIX})", + ) + parser.add_argument( + "--apply", action="store_true", help="actually write to the database" + ) + parser.add_argument( + "--restore", + action="store_true", + help="undo a previous neutralization and reactivate the copies", + ) + config = parser.parse_args() + + if config.restore: + count = restore(config.database, config.prefix) + print(f"✅ -> {count} COW view(s) restored on '{config.database}'.") + return 0 + + if not config.target_version: + parser.error("--target_version is required unless --restore is used") + if not os.path.isdir(config.target_version): + print( + f"❌ Target version directory '{config.target_version}' not found." + ) + return 1 + + lst_at_risk, _, _ = analyse(config.database, config.target_version) + if not lst_at_risk: + print("✅ -> No website COW view to neutralize.") + return 0 + + print( + f"⚠️ {len(lst_at_risk)} website COW view(s) would break the bump to" + f" {config.target_version}:" + ) + lst_view_id = [] + for view_id, key, mode, target_mode, website_id, reason in lst_at_risk: + lst_view_id.append(view_id) + print( + f" - id={view_id} website={website_id} {key}" + f" : {mode} -> {target_mode} ({reason})" + ) + + if not config.apply: + print( + "ℹ Dry-run. Add --apply to rename their key to" + f" '{config.prefix}.' and deactivate them. Reversible with" + " --restore; the arch stays in database." + ) + return 0 + + count = neutralize(config.database, lst_view_id, config.prefix) + print( + f"✅ -> {count} COW view(s) neutralized (key prefixed with" + f" '{config.prefix}.', deactivated). The arch is kept as an archive;" + " use --restore to undo." + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/script/odoo/migration/reset_stale_cow_views.py b/script/odoo/migration/reset_stale_cow_views.py new file mode 100755 index 0000000..13cd44e --- /dev/null +++ b/script/odoo/migration/reset_stale_cow_views.py @@ -0,0 +1,365 @@ +#!/usr/bin/env python3 +# © 2021-2026 TechnoLibre (http://www.technolibre.ca) +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl) + +"""Find the website COW copies that have gone stale, and reset them. + +The problem +----------- +A COW copy freezes the module view it was copied from. Version after version, +the MODULE view is modernised while the copy is not — it is user data, nothing +rewrites it. The copy keeps working right up to the moment a CHILD view does an +``xpath`` onto something the module added, renamed or re-anchored:: + + Element '' + cannot be located in parent view + +Observed on ``web.layout`` during 14.0 -> 15.0: the copy dated from 12.0, where +the script tag had no ``id`` and QWeb still said ``t-raw``. It had survived +until then only because the 14.0 module xpath carried a fallback +(``//head/script[@id='…'] | //head/script[last()]``) that 15.0 removed. + +What this checks +---------------- +Not a heuristic on deprecated syntax, and not an absolute one either: the +DRIFT between a copy and its module twin. For every active child of a COW +copy, each ``xpath`` expression is resolved twice — against the module twin +and against the copy. An expression the twin can satisfy and the copy cannot +is an anchor the copy has lost. + +The comparison has to be differential. Odoo resolves an xpath against the +COMBINED arch of the whole inheritance chain, so « does not resolve in this +parent » proves nothing on its own — a child of ``website.layout`` targets +``//header``, which comes from an ancestor. + +When it bites +------------- +A reported copy is not necessarily failing right now. Odoo re-validates a COW +copy when its module twin is REWRITTEN — which is what a version bump does — +or when the page is rendered for that website. So a finding is a breakage +already present in the data, waiting for the next bump to surface it. That is +precisely when it is cheap to fix. + +Resetting +--------- +``--reset`` copies the MODULE arch over the stale copy. The previous arch is +always written to a backup file first, and the diff is printed, because a copy +can hold a genuine customisation — one line among fifty of drift. Read the +diff, reset, then re-apply what mattered as an INHERITING view rather than a +full copy, so the next version bump cannot make it stale again. + +Plain psql on purpose: this runs on databases whose Odoo registry does not +load, which is precisely when it is needed. +""" + +import argparse +import datetime +import difflib +import json +import os +import re +import subprocess +import sys + + +def run_psql(database, sql): + """Run a statement and return stdout, raising on failure.""" + result = subprocess.run( + ["psql", "-d", database, "-tAc", sql], + capture_output=True, + text=True, + ) + if result.returncode: + raise RuntimeError( + f"Query failed on '{database}': {result.stderr.strip()}" + ) + return result.stdout + + +def normalise_arch(value): + """The arch as a string, whatever the column type. + + Odoo stores ``arch_db`` as text up to 15.0 and as jsonb (one entry per + language) from 16.0. Casting to text gives ``{"en_US": "]*\bexpr=([\"'])(.*?)\1", re.S) + + +def child_xpaths(arch): + """The xpath expressions a view applies to its parent.""" + return [match.group(2) for match in XPATH_RE.finditer(arch)] + + +def require_lxml(): + """The XPath engine, or a loud failure. + + Never degrade to « everything resolves » when lxml is missing: a checker + that cannot check must not answer « all clean ». Odoo's own venvs all ship + lxml; the bare system python3 usually does not. + """ + try: + from lxml import etree + + return etree + except ImportError: + sys.exit( + "❌ lxml is required to resolve the xpath expressions.\n" + " Run this with an interpreter that has it, e.g.\n" + " ./.venv.erplibre/bin/python3 " + os.path.relpath(__file__) + ) + + +def resolve(etree, arch, expr): + """True if `expr` matches something in `arch`. + + An arch that does not parse, or an expression lxml cannot evaluate, counts + as resolvable: this tool reports views that will CERTAINLY break, and must + never invent a failure it cannot substantiate. + """ + try: + tree = etree.fromstring(arch.encode("utf-8")) + except etree.XMLSyntaxError: + return True + try: + return bool(tree.xpath(expr)) + except etree.XPathEvalError: + return True + + +def analyse(database): + """[(copy, module_twin, [(child_id, failing_expr), ...])] — the COW copies + that have DRIFTED away from their module twin. + + The test is differential, and it has to be. Odoo resolves an xpath against + the COMBINED arch of the whole inheritance chain, not against the parent's + own arch, so « does not resolve here » proves nothing on its own: a child + of ``website.layout`` legitimately targets ``//header``, which comes from + an ancestor. Comparing the copy with its module twin removes that whole + class of noise: + + resolves in the module twin, not in the copy -> the copy lost an + anchor: real drift + resolves in neither -> the anchor lives + further up the chain + resolves in both -> nothing to see + + Inactive children are skipped: Odoo never applies them, so they cannot + break a load. + """ + etree = require_lxml() + views = fetch_views(database) + module_by_key = { + v["key"]: v + for v in views.values() + if v["website_id"] is None and v["key"] + } + children = {} + for view in views.values(): + if view["inherit_id"]: + children.setdefault(view["inherit_id"], []).append(view) + + findings = [] + for view in sorted(views.values(), key=lambda v: v["id"]): + twin = module_by_key.get(view["key"]) + if view["website_id"] is None or twin is None: + continue + broken = [] + for child in children.get(view["id"], []): + if not child.get("active", True): + continue + for expr in child_xpaths(child["arch"]): + if resolve(etree, twin["arch"], expr) and not resolve( + etree, view["arch"], expr + ): + broken.append((child["id"], expr)) + if broken: + findings.append((view, twin, broken)) + return findings + + +def show_diff(module_view, cow_view): + """Module arch vs copy: what the copy would gain and lose on a reset.""" + diff = difflib.unified_diff( + module_view["arch"].splitlines(), + cow_view["arch"].splitlines(), + fromfile=f"module id={module_view['id']}", + tofile=f"cow id={cow_view['id']}", + lineterm="", + ) + for line in diff: + print(f" {line}") + + +def backup(database, cow_view, directory): + """Store the arch about to be replaced, and return the file path.""" + os.makedirs(directory, exist_ok=True) + stamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") + path = os.path.join( + directory, f"{cow_view['key']}_{cow_view['id']}_{stamp}.json" + ) + with open(path, "w", encoding="utf-8") as fh: + json.dump( + { + "database": database, + "id": cow_view["id"], + "key": cow_view["key"], + "website_id": cow_view["website_id"], + "saved_at": stamp, + "arch_db": cow_view["arch"], + }, + fh, + indent=2, + ensure_ascii=False, + ) + return path + + +def reset(database, cow_view, module_view): + """Copy the module arch over the stale copy. Returns rows updated.""" + sql = ( + "UPDATE ir_ui_view c SET arch_db = m.arch_db " + "FROM ir_ui_view m " + f"WHERE c.id = {cow_view['id']} AND m.id = {module_view['id']}" + ) + run_psql(database, sql) + return 1 + + +def main(): + parser = argparse.ArgumentParser( + description=( + "Report the website COW copies whose children can no longer be" + " applied, and optionally reset them onto the module view." + ) + ) + parser.add_argument("-d", "--database", required=True) + parser.add_argument( + "--reset", + metavar="KEY", + action="append", + default=[], + help="reset this key onto its module view ('all' for every finding)", + ) + parser.add_argument( + "--apply", + action="store_true", + help="really write; without it --reset only shows what it would do", + ) + parser.add_argument( + "--backup-dir", + default=None, + help="where the replaced arch is saved" + " (default private/odoo/migration//cow_reset)", + ) + config = parser.parse_args() + + findings = analyse(config.database) + if not findings: + print("✅ No COW copy has drifted from its module view.") + return 0 + + print( + f"⚠️ {len(findings)} COW copy(ies) drifted from their module view" + f" in {config.database}" + ) + print( + " Odoo surfaces this when the module view is rewritten (a version" + " bump) or when the page is rendered.\n" + ) + for cow_view, module_view, broken in findings: + twin = ( + f"module id={module_view['id']}" + if module_view + else "NO module view with this key" + ) + print( + f" id={cow_view['id']} key={cow_view['key']}" + f" website_id={cow_view['website_id']} [{twin}]" + ) + for child_id, expr in broken: + print(f" child {child_id} cannot apply: {expr}") + if module_view: + show_diff(module_view, cow_view) + print() + + if not config.reset: + print( + "Nothing changed. Re-run with --reset --apply to reset a" + " copy onto its module view." + ) + return 1 + + wanted = set(config.reset) + directory = config.backup_dir or os.path.join( + "private", "odoo", "migration", config.database, "cow_reset" + ) + done = 0 + for cow_view, module_view, _broken in findings: + if "all" not in wanted and cow_view["key"] not in wanted: + continue + if not module_view: + print( + f"⏭ {cow_view['key']}: no module view to reset onto," + " skipped." + ) + continue + if not config.apply: + print( + f"[dry-run] would reset id={cow_view['id']}" + f" ({cow_view['key']}) onto id={module_view['id']}" + ) + continue + path = backup(config.database, cow_view, directory) + reset(config.database, cow_view, module_view) + done += 1 + print(f"✅ reset id={cow_view['id']} ({cow_view['key']})") + print(f" previous arch saved to {path}") + if config.apply and done: + print( + f"\n{done} copy(ies) reset. Re-apply any real customisation as an" + " INHERITING view, not a copy, so it cannot go stale again." + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/script/odoo/migration/snapshot_cow_views.py b/script/odoo/migration/snapshot_cow_views.py new file mode 100755 index 0000000..a8f839b --- /dev/null +++ b/script/odoo/migration/snapshot_cow_views.py @@ -0,0 +1,213 @@ +#!/usr/bin/env python3 +# © 2021-2026 TechnoLibre (http://www.technolibre.ca) +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl) + +"""Snapshot the website COW views, and diff two snapshots. + +Why +--- +A version bump rewrites website views in ways nobody announces: OpenUpgrade +converts Bootstrap markup on every ``website_id IS NOT NULL`` view, modules +rewrite the copies they own, and some copies simply disappear. Without a +before/after record, "the site looks wrong" is unanswerable. + +Taking a snapshot before and after each jump turns that into a diff: which copy +lost its arch, which changed mode, which was renamed, which vanished. + +Snapshots hold customer template content, so they belong under ``private/`` +and are never versioned. + +Usage:: + + snapshot_cow_views.py -d --label before_13 + snapshot_cow_views.py -d --label after_13 + snapshot_cow_views.py --diff +""" + +import argparse +import datetime +import hashlib +import json +import os +import subprocess +import sys + +# Columns worth recording. ir_ui_view does not expose the same set across 12.0 +# to 18.0, so the query keeps only those that actually exist. +WANTED_COLUMN = [ + "id", + "key", + "name", + "type", + "mode", + "active", + "priority", + "website_id", + "inherit_id", + "arch_fs", + "arch_updated", +] +DEFAULT_DIR = os.path.join("private", "odoo", "migration") + + +def run_psql(database, sql): + """Run a read-only query and return stdout.""" + result = subprocess.run( + ["psql", "-d", database, "-tAc", sql], + capture_output=True, + text=True, + ) + if result.returncode: + raise RuntimeError( + f"Query failed on '{database}': {result.stderr.strip()}" + ) + return result.stdout + + +def existing_columns(database): + """Column names of ir_ui_view present in this database.""" + output = run_psql( + database, + "SELECT column_name FROM information_schema.columns" + " WHERE table_name = 'ir_ui_view';", + ) + return {line.strip() for line in output.splitlines() if line.strip()} + + +def collect(database): + """Return the list of COW views as plain dicts, arch included. + + The rows come back as JSON straight from Postgres: an arch holds newlines + and pipes, so no hand-made separator survives it. + """ + available = existing_columns(database) + lst_column = [name for name in WANTED_COLUMN if name in available] + select = ", ".join(lst_column) + ", arch_db::text AS arch_db" + output = run_psql( + database, + "SELECT COALESCE(json_agg(row_to_json(t)), '[]'::json) FROM (" + f" SELECT {select} FROM ir_ui_view" + " WHERE website_id IS NOT NULL ORDER BY id) t;", + ) + lst_view = json.loads(output or "[]") + for view in lst_view: + arch = view.pop("arch_db", None) or "" + view["arch_md5"] = hashlib.md5(arch.encode("utf-8")).hexdigest() + view["arch_len"] = len(arch) + view["arch_db"] = arch + return lst_view + + +def save(database, label, output_dir): + """Write a snapshot and return its path.""" + directory = output_dir or os.path.join( + DEFAULT_DIR, database, "cow_snapshots" + ) + os.makedirs(directory, exist_ok=True) + lst_view = collect(database) + payload = { + "database": database, + "label": label, + "taken_at": datetime.datetime.now().isoformat(timespec="seconds"), + "count": len(lst_view), + "views": lst_view, + } + file_path = os.path.join(directory, f"{label}.json") + with open(file_path, "w", encoding="utf-8") as f: + json.dump(payload, f, indent=2, ensure_ascii=False) + print(f"✅ -> {len(lst_view)} COW view(s) recorded in {file_path}") + return file_path + + +def load(file_path): + with open(file_path, "r", encoding="utf-8") as f: + return json.load(f) + + +def diff(path_before, path_after): + """Print what changed between two snapshots.""" + before = load(path_before) + after = load(path_after) + map_before = {view["id"]: view for view in before["views"]} + map_after = {view["id"]: view for view in after["views"]} + + removed = sorted(set(map_before) - set(map_after)) + added = sorted(set(map_after) - set(map_before)) + common = sorted(set(map_before) & set(map_after)) + + print( + f"📊 {before.get('label')} ({before.get('count')} views)" + f" -> {after.get('label')} ({after.get('count')} views)" + ) + + if removed: + print(f"❌ {len(removed)} COW view(s) disappeared:") + for view_id in removed: + view = map_before[view_id] + print(f" - id={view_id} {view.get('key')}") + + if added: + print(f"➕ {len(added)} COW view(s) appeared:") + for view_id in added: + view = map_after[view_id] + print(f" - id={view_id} {view.get('key')} ({view.get('mode')})") + + lst_changed = [] + for view_id in common: + old, new = map_before[view_id], map_after[view_id] + lst_field = [] + for field in ("key", "mode", "inherit_id", "active", "arch_md5"): + if old.get(field) != new.get(field): + if field == "arch_md5": + lst_field.append( + f"arch rewritten ({old.get('arch_len')} ->" + f" {new.get('arch_len')} chars)" + ) + else: + lst_field.append( + f"{field}: {old.get(field)} -> {new.get(field)}" + ) + if lst_field: + lst_changed.append((view_id, new.get("key"), lst_field)) + + if lst_changed: + print(f"✏️ {len(lst_changed)} COW view(s) changed:") + for view_id, key, lst_field in lst_changed: + print(f" - id={view_id} {key}") + for change in lst_field: + print(f" {change}") + + if not (removed or added or lst_changed): + print("✅ -> No change on the website COW views.") + return 0 + + +def main(): + parser = argparse.ArgumentParser( + description="Snapshot website COW views, or diff two snapshots." + ) + parser.add_argument("-d", "--database", help="database to snapshot") + parser.add_argument( + "-l", "--label", help="snapshot name, e.g. before_13 or after_13" + ) + parser.add_argument( + "-o", "--output_dir", help="where to write (default: private/...)" + ) + parser.add_argument( + "--diff", + nargs=2, + metavar=("BEFORE", "AFTER"), + help="compare two snapshot files instead of taking one", + ) + config = parser.parse_args() + + if config.diff: + return diff(*config.diff) + if not config.database or not config.label: + parser.error("--database and --label are required to take a snapshot") + save(config.database, config.label, config.output_dir) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/script/todo/migration_form.py b/script/todo/migration_form.py new file mode 100644 index 0000000..98a150c --- /dev/null +++ b/script/todo/migration_form.py @@ -0,0 +1,186 @@ +#!/usr/bin/env python3 +# © 2021-2026 TechnoLibre (http://www.technolibre.ca) +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl) +"""Écran de reprise de migration Odoo, en TUI. + +Pendant de `qemu_deploy_form` pour l'outil de migration. Deux interfaces +posent la MÊME première question — « où en est-on, et par où reprend-on ? » — +et renvoient les MÊMES chaînes de réponse (« c », « n », « r », « q », +« 0 »..« 4 », « 4. ») que `TodoUpgrade.apply_resume_answer` traduit +en progression. La décision est donc écrite une seule fois. + +- run_resume_tui(ctx, run_app=True) : renvoie la réponse, ou None pour + retomber sur les invites en ligne. + +`ctx` vient de `TodoUpgrade.resume_context()` : pure donnée, aucun accès à la +base ni au disque depuis l'affichage. +""" +from __future__ import annotations + +try: + from script.todo.todo_i18n import t +except Exception: # pragma: no cover - repli si i18n indisponible + + def t(key: str) -> str: + return key + + +def version_line(versions): + """« 13✓ 14✓ 15 16 17 18 » — l'avancement des montées de version.""" + return " ".join( + f"{v['version']}{'✓' if v['done'] else ''}" for v in versions + ) + + +def next_version(versions): + """Première version pas encore migrée : celle que « continuer » reprend.""" + for item in versions: + if not item["done"]: + return item["version"] + return None + + +def run_resume_tui(ctx, run_app: bool = True): + """Écran de reprise. Renvoie la réponse choisie, ou None si annulé. + `run_app=False` renvoie l'instance sans la lancer (tests headless).""" + from textual.app import App, ComposeResult + from textual.containers import Horizontal, Vertical + from textual.widgets import ( + Button, + DataTable, + Footer, + Header, + OptionList, + Static, + ) + from textual.widgets.option_list import Option + + result = {"answer": None} + + class Resume(App): + CSS = """ + #head { height: auto; padding: 0 1; color: $text-muted; } + #steps { height: auto; max-height: 12; border: solid $accent; } + #bumps { height: auto; max-height: 10; border: solid $panel; } + .grouptitle { color: $accent; text-style: bold; padding: 1 1 0 1; } + #actions { height: auto; padding: 1 1 0 1; } + #hint { height: auto; color: $text-muted; padding: 0 1; } + """ + BINDINGS = [ + ("c", "cont", t("Continue where it stopped")), + ("n", "new", t("New migration, erase everything")), + ("r", "keep_zip", t("Keep the zip only")), + ("q", "quit_nothing", t("Quit without doing anything")), + ("escape", "quit_nothing", t("Quit without doing anything")), + ] + + def compose(self) -> ComposeResult: + yield Header() + yield Static( + f" {t('File'):<9}: {ctx['file']}\n" + f" {t('Database'):<9}: {ctx['database']}" + f" · {t('Target')} : {ctx['target']}\n" + f" {t('Started'):<9}: {ctx['started']}", + id="head", + ) + yield Static(f"{t('Steps')}", classes="grouptitle") + yield DataTable(id="steps") + if ctx["versions"]: + yield Static( + f"{t('Version bumps')} ({version_line(ctx['versions'])})", + classes="grouptitle", + ) + yield OptionList(id="bumps") + with Vertical(): + with Horizontal(id="actions"): + yield Button( + t("Continue where it stopped"), + variant="primary", + id="a_cont", + ) + yield Button(t("New migration"), id="a_new") + yield Button(t("Keep the zip only"), id="a_keep") + yield Button(t("Quit"), id="a_quit") + yield Static( + f" {t('Enter on a step or a version = replay from there')}", + id="hint", + ) + yield Footer() + + def on_mount(self) -> None: + self.title = t("Migration in progress") + table = self.query_one("#steps", DataTable) + table.cursor_type = "row" + table.add_columns("", "", t("Step"), t("Detail")) + for item in ctx["steps"]: + table.add_row( + f"[{item['step']}]", + item["icon"], + item["label"], + item["detail"], + ) + # Curseur sur la première étape inachevée : c'est là que ça a + # calé, donc là qu'on veut probablement rejouer. + for index, item in enumerate(ctx["steps"]): + if item["icon"] != "✅": + table.move_cursor(row=index) + break + if ctx["versions"]: + bumps = self.query_one("#bumps", OptionList) + for item in ctx["versions"]: + mark = "✓" if item["done"] else " " + bumps.add_option( + Option( + f" {mark} Odoo {item['version']}.0 — " + f"{t('rebuilds the intermediate database')}", + id=str(item["version"]), + ) + ) + upcoming = next_version(ctx["versions"]) + if upcoming is not None: + bumps.highlighted = [ + v["version"] for v in ctx["versions"] + ].index(upcoming) + + # -- choix ------------------------------------------------------ # + def _answer(self, value): + result["answer"] = value + self.exit() + + def on_data_table_row_selected(self, event) -> None: + index = event.cursor_row + if 0 <= index < len(ctx["steps"]): + self._answer(str(ctx["steps"][index]["step"])) + + def on_option_list_option_selected(self, event) -> None: + self._answer(f"4.{event.option.id}") + + def on_button_pressed(self, event) -> None: + mapping = { + "a_cont": "c", + "a_new": "n", + "a_keep": "r", + "a_quit": "q", + } + value = mapping.get(event.button.id) + if value: + self._answer(value) + + def action_cont(self) -> None: + self._answer("c") + + def action_new(self) -> None: + self._answer("n") + + def action_keep_zip(self) -> None: + self._answer("r") + + def action_quit_nothing(self) -> None: + self._answer("q") + + app = Resume() + app._result = result # lecture par les tests headless + if not run_app: + return app + app.run() + return result["answer"] diff --git a/script/todo/migration_stats.py b/script/todo/migration_stats.py new file mode 100644 index 0000000..715d7cc --- /dev/null +++ b/script/todo/migration_stats.py @@ -0,0 +1,201 @@ +#!/usr/bin/env python3 +# © 2021-2026 TechnoLibre (http://www.technolibre.ca) +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl) +"""Statistiques d'une migration Odoo, en lecture seule. + +Ne touche NI la base NI le fichier de progression : tout se déduit du journal +de migration et des traces laissées sous `private/odoo/migration//`. +C'est ce qui permet de consulter l'état d'une migration en cours depuis une +autre session sans risquer de la perturber. + +`compute()` reçoit le contexte déjà construit par +`TodoUpgrade.resume_context()` — étapes et montées de version — pour ne pas +réimplémenter une seconde fois la lecture des clés « state_* », qui +divergerait de l'écran de reprise. +""" +from __future__ import annotations + +import datetime +import glob +import json +import os + +try: + from script.todo.todo_i18n import t +except Exception: # pragma: no cover - repli si i18n indisponible + + def t(key: str) -> str: + return key + + +def _as_int(value): + """Clé de version en entier. JSON transforme les clés numériques en + chaînes : « 13 » et 13 désignent la même version.""" + try: + return int(float(value)) + except (TypeError, ValueError): + return None + + +def fmt_delay(start, end): + """Écart lisible entre deux horodatages « str(datetime) ».""" + try: + a = datetime.datetime.fromisoformat(str(start)) + b = datetime.datetime.fromisoformat(str(end)) + except (TypeError, ValueError): + return "?" + secs = int(abs((b - a).total_seconds())) + days, rest = divmod(secs, 86400) + hours, rest = divmod(rest, 3600) + minutes = rest // 60 + if days: + return f"{days} j {hours:02d} h" + if hours: + return f"{hours} h {minutes:02d} min" + return f"{minutes} min" + + +def module_evolution(dct_progression): + """[(version, nb_modules, delta)] du plus ancien au plus récent. + + Montre où les modules disparaissent : un saut qui en perd 15 d'un coup + n'a pas le même sens qu'un saut qui n'en perd aucun. + """ + raw = dct_progression.get("dct_module_per_version") or {} + rows = [] + for key, value in raw.items(): + version = _as_int(key) + if version is not None and isinstance(value, list): + rows.append((version, len(value))) + rows.sort() + out = [] + previous = None + for version, count in rows: + out.append( + (version, count, None if previous is None else count - previous) + ) + previous = count + return out + + +def cow_snapshots(private_dir, database_name): + """Instantanés de vues COW enregistrés, du plus ancien au plus récent.""" + directory = os.path.join(private_dir, database_name, "cow_snapshots") + out = [] + for path in sorted(glob.glob(os.path.join(directory, "*.json"))): + try: + with open(path, encoding="utf-8") as fh: + data = json.load(fh) + except (OSError, ValueError): + continue + out.append( + { + "label": data.get("label") or os.path.basename(path)[:-5], + "count": data.get("count"), + "taken_at": data.get("taken_at") or "?", + "path": path, + } + ) + out.sort(key=lambda item: item["taken_at"]) + return out + + +def fix_hooks(ctx, global_dir): + """Correctifs de migration disponibles, et lesquels ont tourné.""" + applied = ctx.get("_fix_applied") or [] + out = [] + for index, item in enumerate(ctx.get("versions") or []): + target = item["version"] + stem = f"fix_migration_odoo{(target - 1) * 10}_to_odoo{target * 10}" + found = [ + os.path.basename(p) + for ext in (".sql", ".py") + for p in [os.path.join(global_dir, stem + ext)] + if os.path.exists(p) + ] + if not found: + continue + out.append( + { + "version": target, + "file": found[0], + "applied": bool(index < len(applied) and applied[index]), + } + ) + return out + + +def journal(dct_progression): + """Commandes exécutées et décisions annotées (les lignes « # »).""" + lst = dct_progression.get("command_executed") or [] + comments = [ + c[2:].strip() for c in lst if isinstance(c, str) and c.startswith("# ") + ] + commands = [ + c for c in lst if isinstance(c, str) and not c.startswith("# ") + ] + return {"commands": commands, "comments": comments} + + +def compute( + dct_progression, + ctx, + database_name, + read_uninstall, + private_dir, + global_dir, +): + """Rassemble toutes les statistiques. + + `read_uninstall(version, base)` est la lecture de liste de TodoUpgrade : + elle résout ELLE-MÊME ses chemins privé puis global, ce qui garantit + qu'on affiche exactement la liste qui serait appliquée. `private_dir` ne + sert donc qu'aux instantanés COW, et `global_dir` qu'aux correctifs.""" + versions = [item["version"] for item in (ctx.get("versions") or [])] + uninstall = {} + for target in versions: + try: + _lst, detail = read_uninstall(target - 1, database_name) + except Exception: + detail = [] + if detail: + uninstall[target] = detail + + evolution = module_evolution(dct_progression) + origin = dct_progression.get("lst_module_per_version_origin") or [] + return { + "delay": fmt_delay( + dct_progression.get("date_create"), + dct_progression.get("date_update"), + ), + "updated": dct_progression.get("date_update") or "?", + "evolution": evolution, + "origin_count": len(origin) if isinstance(origin, list) else 0, + "missing": dct_progression.get("lst_module_missing") or [], + "duplicate": dct_progression.get("lst_module_duplicate") or [], + "uninstall": uninstall, + "removed_total": sum(len(v) for v in uninstall.values()), + "cow": cow_snapshots(private_dir, database_name), + "fixes": fix_hooks( + dict( + ctx, + _fix_applied=dct_progression.get( + "state_4_fix_migration_odoo_lst" + ) + or [], + ), + global_dir, + ), + "journal": journal(dct_progression), + } + + +def flat_module_list(uninstall): + """Tous les modules supprimés, dédupliqués, prêts à copier-coller.""" + seen = [] + for detail in uninstall.values(): + for item in detail: + name = item[0] if isinstance(item, (list, tuple)) else item + if name not in seen: + seen.append(name) + return seen diff --git a/script/todo/todo_i18n.py b/script/todo/todo_i18n.py index e17e94f..5b8eba3 100644 --- a/script/todo/todo_i18n.py +++ b/script/todo/todo_i18n.py @@ -1687,6 +1687,200 @@ TRANSLATIONS = { "fr": "via", "en": "via", }, + "Odoo migration interface": { + "fr": "🚚 Interface de la migration Odoo", + "en": "🚚 Odoo migration interface", + }, + "modules of the list have no code in the active Odoo:": { + "fr": "modules de la liste n'ont plus de code dans l'Odoo actif :", + "en": "modules of the list have no code in the active Odoo:", + }, + "Odoo cannot uninstall a module whose code is gone;": { + "fr": "Odoo ne peut pas désinstaller un module dont le code a" + " disparu ;", + "en": "Odoo cannot uninstall a module whose code is gone;", + }, + "one of them fails the whole uninstall command.": { + "fr": "un seul d'entre eux fait échouer toute la désinstallation.", + "en": "one of them fails the whole uninstall command.", + }, + "are present and can be uninstalled:": { + "fr": "sont présents et peuvent être désinstallés :", + "en": "are present and can be uninstalled:", + }, + "No module of the list is present.": { + "fr": "Aucun module de la liste n'est présent.", + "en": "No module of the list is present.", + }, + "Uninstall the present ones, skip the missing": { + "fr": "Désinstaller les présents, ignorer les manquants", + "en": "Uninstall the present ones, skip the missing", + }, + "Try the whole list anyway (it will fail)": { + "fr": "Tout tenter quand même (ça échouera)", + "en": "Try the whole list anyway (it will fail)", + }, + "Uninstall nothing, continue": { + "fr": "Ne rien désinstaller, continuer", + "en": "Uninstall nothing, continue", + }, + "Nothing uninstalled.": { + "fr": "Rien de désinstallé.", + "en": "Nothing uninstalled.", + }, + "Check the COW views that drifted": { + "fr": "Vérifier les vues COW en retard sur leur vue module", + "en": "Check the COW views that drifted", + }, + "Tool not found": { + "fr": "Outil introuvable", + "en": "Tool not found", + }, + "To reset one of them onto its module view:": { + "fr": "Pour réinitialiser l'une d'elles sur sa vue module :", + "en": "To reset one of them onto its module view:", + }, + "Read the diff first: a copy can hold a customisation.": { + "fr": "Lire le diff d'abord : une copie peut porter une" + " personnalisation.", + "en": "Read the diff first: a copy can hold a customisation.", + }, + "Migration statistics (read-only)": { + "fr": "📊 Statistiques de la migration (lecture seule)", + "en": "📊 Migration statistics (read-only)", + }, + "Choice (0-3, default 1): ": { + "fr": "Choix (0-3, défaut 1) : ", + "en": "Choice (0-3, default 1): ", + }, + "Migration statistics": { + "fr": "Statistiques de la migration", + "en": "Migration statistics", + }, + "elapsed": { + "fr": "écoulé", + "en": "elapsed", + }, + "Level reached": { + "fr": "Niveau atteint", + "en": "Level reached", + }, + "Modules": { + "fr": "Modules", + "en": "Modules", + }, + "modules": { + "fr": "modules", + "en": "modules", + }, + "At the start": { + "fr": "Au départ", + "en": "At the start", + }, + "Removed in total": { + "fr": "Supprimés au total", + "en": "Removed in total", + }, + "Reported missing": { + "fr": "Signalés manquants", + "en": "Reported missing", + }, + "Duplicated": { + "fr": "En double", + "en": "Duplicated", + }, + "Migration fixes": { + "fr": "Correctifs de migration", + "en": "Migration fixes", + }, + "COW views": { + "fr": "Vues COW", + "en": "COW views", + }, + "views": { + "fr": "vues", + "en": "views", + }, + "no snapshot": { + "fr": "aucun instantané", + "en": "no snapshot", + }, + "Journal": { + "fr": "Journal", + "en": "Journal", + }, + "commands": { + "fr": "commandes", + "en": "commands", + }, + "recorded decisions": { + "fr": "décisions annotées", + "en": "recorded decisions", + }, + "Removed modules, with their reason": { + "fr": "Modules supprimés, avec leur justification", + "en": "Removed modules, with their reason", + }, + "Removed modules, comma-separated (copy)": { + "fr": "Modules supprimés, séparés par des virgules (à copier)", + "en": "Removed modules, comma-separated (copy)", + }, + "COW views: snapshots and differences": { + "fr": "Vues COW : instantanés et différences", + "en": "COW views: snapshots and differences", + }, + "Recorded decisions (journal)": { + "fr": "Décisions annotées (journal)", + "en": "Recorded decisions (journal)", + }, + "Executed commands (last 30)": { + "fr": "Commandes exécutées (30 dernières)", + "en": "Executed commands (last 30)", + }, + "nothing recorded": { + "fr": "rien d'annoté", + "en": "nothing recorded", + }, + "Need two snapshots to diff.": { + "fr": "Il faut deux instantanés pour comparer.", + "en": "Need two snapshots to diff.", + }, + "Diff which two? (e.g. 1,2 — blank to skip)": { + "fr": "Comparer lesquels ? (ex. 1,2 — vide pour passer)", + "en": "Diff which two? (e.g. 1,2 — blank to skip)", + }, + "No migration in progress to resume.": { + "fr": "Aucune migration en cours à reprendre.", + "en": "No migration in progress to resume.", + }, + "Quit without doing anything": { + "fr": "Quitter sans rien faire", + "en": "Quit without doing anything", + }, + "Version bumps": { + "fr": "Montées de version", + "en": "Version bumps", + }, + "Step": { + "fr": "Étape", + "en": "Step", + }, + "Detail": { + "fr": "Détail", + "en": "Detail", + }, + "Enter on a step or a version = replay from there": { + "fr": "Entrée sur une étape ou une version = rejouer depuis là", + "en": "Enter on a step or a version = replay from there", + }, + "New migration": { + "fr": "Nouvelle migration", + "en": "New migration", + }, + "Keep the zip only": { + "fr": "Garder seulement le zip", + "en": "Keep the zip only", + }, "Reset all preferences": { "fr": "🧹 Réinitialiser toutes les préférences", "en": "🧹 Reset all preferences", @@ -1952,6 +2146,26 @@ TRANSLATIONS = { "fr": "Échec de la commande : ", "en": "Command failed: ", }, + "Textual is required for this screen.": { + "fr": "Textual est nécessaire pour cet écran.", + "en": "Textual is required for this screen.", + }, + "Install it now? (Y/n): ": { + "fr": "L'installer maintenant ? (O/n, défaut : oui) : ", + "en": "Install it now? (Y/n, default: yes): ", + }, + "Textual is installed.": { + "fr": "Textual est installé.", + "en": "Textual is installed.", + }, + "Installation finished but textual is still missing.": { + "fr": "Installation terminée mais textual reste introuvable.", + "en": "Installation finished but textual is still missing.", + }, + "pip exited with": { + "fr": "pip a retourné", + "en": "pip exited with", + }, "Your distribution may package it as python3-textual.": { "fr": "Votre distribution le fournit peut-être en python3-textual.", "en": "Your distribution may package it as python3-textual.", @@ -3125,6 +3339,85 @@ TRANSLATIONS = { "recorded runs": {"fr": "runs enregistrés", "en": "recorded runs"}, "runs erased": {"fr": "runs effacés", "en": "runs erased"}, "Cancelled.": {"fr": "Annulé.", "en": "Cancelled."}, + # Database migration - resume menu + "Migration in progress": { + "fr": "Migration en cours", + "en": "Migration in progress", + }, + "File": {"fr": "Fichier", "en": "File"}, + "Database": {"fr": "Base", "en": "Database"}, + "Target": {"fr": "Cible", "en": "Target"}, + "Started": {"fr": "Démarrée", "en": "Started"}, + "Steps": {"fr": "Étapes", "en": "Steps"}, + "Prepare the environment": { + "fr": "Préparer l'environnement", + "en": "Prepare the environment", + }, + "Restore and neutralize the database": { + "fr": "Restaurer et neutraliser la base", + "en": "Restore and neutralize the database", + }, + "Update all addons": { + "fr": "Mettre à jour tous les modules", + "en": "Update all addons", + }, + "Clean up before data migration": { + "fr": "Nettoyer avant la migration des données", + "en": "Clean up before data migration", + }, + "Upgrade version by version (OpenUpgrade)": { + "fr": "Monter de version en version (OpenUpgrade)", + "en": "Upgrade version by version (OpenUpgrade)", + }, + "not started": {"fr": "non démarrée", "en": "not started"}, + "done": {"fr": "terminée", "en": "done"}, + "partially done": {"fr": "partielle", "en": "partially done"}, + "version bumps migrated": { + "fr": "montées de version faites", + "en": "version bumps migrated", + }, + "Continue where it stopped": { + "fr": "Continuer là où ça s'est arrêté", + "en": "Continue where it stopped", + }, + "Replay from that step": { + "fr": "Reprendre à partir de cette étape", + "en": "Replay from that step", + }, + "erases the progression of that step and the next ones": { + "fr": "efface la progression de cette étape et des suivantes", + "en": "erases the progression of that step and the next ones", + }, + "New migration, erase everything": { + "fr": "Nouvelle migration, tout effacer", + "en": "New migration, erase everything", + }, + "Keep the zip only, ask every question again": { + "fr": "Garder seulement le zip, reposer toutes les questions", + "en": "Keep the zip only, ask every question again", + }, + "Your choice": {"fr": "Votre choix", "en": "Your choice"}, + "Unknown choice, continuing where it stopped": { + "fr": "Choix inconnu, on continue là où ça s'est arrêté", + "en": "Unknown choice, continuing where it stopped", + }, + "Replaying from step": { + "fr": "Reprise à partir de l'étape", + "en": "Replaying from step", + }, + "Replay the upgrade from version N": { + "fr": "Reprendre la montée à partir de la version N", + "en": "Replay the upgrade from version N", + }, + "rebuilds the intermediate database": { + "fr": "recrée la base intermédiaire", + "en": "rebuilds the intermediate database", + }, + "Unknown version": {"fr": "Version inconnue", "en": "Unknown version"}, + "The progression file is invalid, ignoring it": { + "fr": "Le fichier de progression est invalide, on l'ignore", + "en": "The progression file is invalid, ignoring it", + }, # --- Menu Execute › Analyse --- # L'émoji vit dans la valeur, jamais dans la clé. Deux clés distinctes, # comme « Data » / « Database » : l'entrée de menu porte l'émoji, diff --git a/script/todo/todo_upgrade.py b/script/todo/todo_upgrade.py index d77b6dc..6c37836 100755 --- a/script/todo/todo_upgrade.py +++ b/script/todo/todo_upgrade.py @@ -6,15 +6,25 @@ import datetime import json import logging import os +import re import shutil import sys import zipfile from uuid import uuid4 -from script.todo.version_manager import get_odoo_version import click import todo_file_browser +from script.todo.version_manager import get_odoo_version + +try: + from script.todo.todo_i18n import t +except Exception: # pragma: no cover - fallback when i18n is unavailable + + def t(key: str) -> str: + return key + + new_path = os.path.normpath( os.path.join(os.path.dirname(__file__), "..", "..") ) @@ -43,6 +53,23 @@ FILENAME_ODOO_VERSION = ".odoo-version" LOCAL_MANIFEST = os.path.join( ".repo", "local_manifests", "erplibre_manifest.xml" ) +# Module lists for a version bump. The shared, versioned defaults live under +# script/; the per-database lists live under private/ (mirroring script/, like +# script/todo/todo.json -> private/todo/todo_override.json). Which modules must +# be dropped depends on the database, so that choice is never versioned. +PATH_MIGRATION_GLOBAL = os.path.join("script", "odoo", "migration") +PATH_MIGRATION_PRIVATE = os.path.join("private", "odoo", "migration") +# Steps of the migration, in order. Each one owns the progression keys prefixed +# with « state_ ». Rewinding to a step drops its keys and every later +# one, so the run replays from there. Labels go through t(): the key IS the +# English string, as everywhere else in this project. +MIGRATION_STEP = [ + (0, "Prepare the environment"), + (1, "Restore and neutralize the database"), + (2, "Update all addons"), + (3, "Clean up before data migration"), + (4, "Upgrade version by version (OpenUpgrade)"), +] class TodoUpgrade: @@ -68,6 +95,501 @@ class TodoUpgrade: with open(UPGRADE_DATABASE_CONFIG_LOG, "w") as f: json.dump(self.dct_progression, f, indent=4) + @staticmethod + def read_progression(): + """Return the saved progression, or an empty dict if unreadable.""" + try: + with open(UPGRADE_DATABASE_CONFIG_LOG, "r") as f: + return json.load(f) + except (json.decoder.JSONDecodeError, OSError): + print( + f"⚠️ {t('The progression file is invalid, ignoring it')}:" + f" {UPGRADE_DATABASE_CONFIG_LOG}" + ) + return {} + + @staticmethod + def step_status(dct_progression, step): + """Return (icon, detail) telling how far a migration step went. + + Steps 0 to 3 are plain booleans. Step 4 keeps one list per version, so + its detail reports which version bumps are already migrated. + """ + prefix = f"state_{step}_" + dct_flag = { + key: value + for key, value in dct_progression.items() + if key.startswith(prefix) + } + if not dct_flag: + return "⬜", t("not started") + + if step == 4: + lst_done = dct_progression.get("state_4_upgrade_odoo_lst") or [] + # The data migration list only appears once a bump succeeds, so the + # number of bumps comes from any per-version list (« *_odoo_lst »). + total = max( + [ + len(value) + for key, value in dct_progression.items() + if key.startswith("state_4_") + and key.endswith("_odoo_lst") + and isinstance(value, list) + ] + or [0] + ) + done = sum(1 for item in lst_done if item) + detail = f"{done}/{total} " + t("version bumps migrated") + # Name the versions when the target is known: the list ends on the + # target, so the first bump is target - total + 1. + try: + last = int(float(dct_progression["target_odoo_version"])) + detail += " · " + " ".join( + f"{last - total + 1 + i}" + f"{'✓' if i < len(lst_done) and lst_done[i] else ''}" + for i in range(total) + ) + except (KeyError, TypeError, ValueError): + pass + return ("✅" if total and done == total else "⏳"), detail + + if all(dct_flag.values()): + return "✅", t("done") + return "⏳", t("partially done") + + def resume_context(self, old_dct_progression): + """Everything the resume screen shows, as plain data. + + No I/O: the line-by-line prompt and the TUI both render THIS, so the + two can never describe the migration differently. + """ + migration_file = old_dct_progression.get("migration_file") or "?" + steps = [] + for step, label in MIGRATION_STEP: + icon, detail = self.step_status(old_dct_progression, step) + steps.append( + { + "step": step, + "icon": icon, + "label": t(label), + "detail": detail, + } + ) + lst_version = self.version_bumps(old_dct_progression) + done = old_dct_progression.get("state_4_upgrade_odoo_lst") or [] + return { + "file": os.path.basename(migration_file), + "database": old_dct_progression.get("config_database_name") or "?", + "target": old_dct_progression.get("target_odoo_version") or "?", + "started": old_dct_progression.get("date_create") or "?", + "steps": steps, + "versions": [ + { + "version": version, + "done": bool(i < len(done) and done[i]), + } + for i, version in enumerate(lst_version) + ], + } + + @staticmethod + def print_resume(ctx): + """Render the resume screen on the terminal.""" + print() + print(f"📍 {t('Migration in progress')}") + # Pad in code, not in the translations: the labels differ in length + # between languages and a hardcoded padding misaligns the colons. + print(f" {t('File'):<9}: {ctx['file']}") + print( + f" {t('Database'):<9}: {ctx['database']}" + f" · {t('Target')} : {ctx['target']}" + ) + print(f" {t('Started'):<9}: {ctx['started']}") + print() + print(f" {t('Steps')} :") + for item in ctx["steps"]: + print( + f" [{item['step']}] {item['icon']} " + f"{item['label']:<44} {item['detail']}" + ) + print() + print(f" [c] {t('Continue where it stopped')}") + print( + f" [0-4] {t('Replay from that step')}" + f" ({t('erases the progression of that step and the next ones')})" + ) + if ctx["versions"]: + versions = "/".join(str(v["version"]) for v in ctx["versions"]) + print( + f" [4.N] {t('Replay the upgrade from version N')}" + f" ({versions}) —" + f" {t('rebuilds the intermediate database')}" + ) + print(f" [n] {t('New migration, erase everything')}") + print(f" [r] {t('Keep the zip only, ask every question again')}") + print(f" [q] {t('Quit without doing anything')}") + + def apply_resume_answer(self, old_dct_progression, answer, ctx): + """Turn the answer into (progression, changed), or None to quit. + + THE decision point, shared by both interfaces: the TUI returns the + same answer strings as the prompt, so this logic is written once. + """ + answer = (answer or "").strip().lower() + lst_version = [v["version"] for v in ctx["versions"]] + + if answer in ("", "c"): + return old_dct_progression, False + if answer == "q": + return None + if answer == "n": + return {}, True + if answer == "r": + return { + "migration_file": old_dct_progression.get("migration_file"), + "date_create": old_dct_progression.get("date_create"), + }, True + if answer.startswith("4.") and lst_version: + target = answer.split(".", 1)[1].strip() + if target.isdigit() and int(target) in lst_version: + return ( + self.rewind_version_bump( + old_dct_progression, lst_version.index(int(target)) + ), + True, + ) + print(f"⚠️ {t('Unknown version')} : {target}") + return old_dct_progression, False + if answer.isdigit() and 0 <= int(answer) <= MIGRATION_STEP[-1][0]: + return ( + self.rewind_progression(old_dct_progression, int(answer)), + True, + ) + + print(f"⚠️ {t('Unknown choice, continuing where it stopped')}.") + return old_dct_progression, False + + def prompt_resume(self, old_dct_progression, use_tui=False): + """Show where the migration stands and ask what to do next. + + Returns (progression, changed), or None if the user quits. The old + menu exposed internal key names (« Reuse database without state_4 »), + which said nothing about what would happen; this shows the real state + of every step and lets the user replay from any of them. + """ + ctx = self.resume_context(old_dct_progression) + answer = None + if use_tui: + answer = self.resume_tui(ctx) + if answer is None: + self.print_resume(ctx) + answer = input(f"💬 {t('Your choice')} : ") + return self.apply_resume_answer(old_dct_progression, answer, ctx) + + @staticmethod + def ask_ui(): + """Interface of the migration: TUI, line-by-line prompts, or the + read-only statistics screen. Returns None to leave the tool. + + The preference can settle it in advance (TODO > Configuration); + « ask » asks. Same contract as the QEMU deployment — except for + « stats », which is never a stored default: it does nothing, so + landing there every time would only be in the way. + """ + try: + from script.todo import todo_prefs + + pref = todo_prefs.get("migration_ui") + except Exception: + pref = "ask" + if pref in ("tui", "cli"): + return pref + print(f"\n{t('Interface:')}") + print(f" [1] {t('TUI form')} *") + print(f" [2] {t('Classic questions (line by line)')}") + print(f" [3] {t('Migration statistics (read-only)')}") + print(f" [0] {t('Cancel')}") + print(f" {t('(change the default in TODO > Configuration)')}") + answer = input(t("Choice (0-3, default 1): ")).strip() + return {"0": None, "2": "cli", "3": "stats"}.get(answer, "tui") + + @staticmethod + def database_from_command(cmd): + """Nom de base visé par une commande, ou "" si indécelable. + + Sert à proposer le bon outil au bon moment quand une commande échoue. + On reconnaît les trois formes du dépôt : « -d », « --database + », et l'argument positionnel des scripts addons + (`update_addons_all.sh `, `install_addons*.sh `). + """ + if not cmd: + return "" + match = re.search(r"(?:^|\s)(?:-d|--database)[=\s]+([\w.-]+)", cmd) + if match: + return match.group(1) + match = re.search( + r"\./script/addons/\w+\.sh\s+([\w.-]+)", + cmd, + ) + return match.group(1) if match else "" + + def check_stale_cow_views(self, database_name): + """Lance le détecteur de copies COW en retard sur leur vue module. + + Purement consultatif : il n'écrit rien sans « --reset … --apply », que + l'on propose seulement après avoir montré le diff — réinitialiser une + copie peut effacer une personnalisation réelle.""" + script_path = os.path.join( + PATH_MIGRATION_GLOBAL, "reset_stale_cow_views.py" + ) + if not os.path.exists(script_path): + print(f"⚠️ {t('Tool not found')}: {script_path}") + return + status, _cmd = self.todo_upgrade_execute( + f"{PYTHON_BIN} ./{script_path} -d {database_name}", + wait_at_error=False, + ) + if status: + # Sortie 1 = des écarts ont été trouvés (le script les a listés). + warn = t("Read the diff first: a copy can hold a customisation.") + print(f"\n💡 {t('To reset one of them onto its module view:')}") + print( + f" {PYTHON_BIN} ./{script_path} -d {database_name}" + f" --reset --apply" + ) + print(f" {warn}") + + def show_stats(self): + """Écran de statistiques, en lecture seule : rien n'est écrit, ni + dans la base ni dans le journal de migration.""" + from script.todo import migration_stats as ms + + if not os.path.exists(UPGRADE_DATABASE_CONFIG_LOG): + print(f"\nℹ️ {t('No migration in progress to resume.')}") + return + dct = self.read_progression() + if not dct: + print(f"\nℹ️ {t('No migration in progress to resume.')}") + return + ctx = self.resume_context(dct) + database_name = dct.get("config_database_name") or "" + stats = ms.compute( + dct, + ctx, + database_name, + self.read_uninstall_module_list, + PATH_MIGRATION_PRIVATE, + PATH_MIGRATION_GLOBAL, + ) + + while True: + self.print_stats(ctx, stats) + print(f"\n [1] {t('Removed modules, with their reason')}") + print(f" [2] {t('Removed modules, comma-separated (copy)')}") + print(f" [3] {t('COW views: snapshots and differences')}") + print(f" [4] {t('Recorded decisions (journal)')}") + print(f" [5] {t('Executed commands (last 30)')}") + print(f" [0] {t('Back')}") + answer = input(f"💬 {t('Your choice')} : ").strip() + if answer in ("", "0"): + return + if answer == "1": + for version, detail in sorted(stats["uninstall"].items()): + print(f"\n── {version - 1}.0 → {version}.0 ──") + self.print_uninstall_reason(detail) + elif answer == "2": + flat = ms.flat_module_list(stats["uninstall"]) + print(f"\n{len(flat)} {t('modules')} :\n") + print(",".join(flat)) + elif answer == "3": + self.stats_cow(stats, database_name) + elif answer == "4": + for line in stats["journal"]["comments"]: + print(f" · {line}") + if not stats["journal"]["comments"]: + print(f" {t('nothing recorded')}") + elif answer == "5": + for line in stats["journal"]["commands"][-30:]: + print(f" $ {line}") + else: + print(f"⚠️ {t('Unknown choice, continuing where it stopped')}.") + + @staticmethod + def print_stats(ctx, stats): + """Rend le tableau de bord de la migration.""" + print(f"\n📊 {t('Migration statistics')}") + print(f" {t('File'):<11}: {ctx['file']}") + print( + f" {t('Database'):<11}: {ctx['database']}" + f" · {t('Target')} : {ctx['target']}" + ) + print( + f" {t('Started'):<11}: {ctx['started']}" + f" · {t('elapsed')} {stats['delay']}" + ) + + print(f"\n── {t('Level reached')} ──") + done = sum(1 for v in ctx["versions"] if v["done"]) + total = len(ctx["versions"]) or 1 + line = " " + for item in ctx["versions"]: + mark = "✅" if item["done"] else "⬜" + line += f"{item['version'] - 1}.0→{item['version']}.0 {mark} " + print(line) + print( + f" {done}/{len(ctx['versions'])} " + f"{t('version bumps migrated')} ({done * 100 // total} %)" + ) + print( + " " + + " ".join(f"[{s['step']}]{s['icon']}" for s in ctx["steps"]) + ) + + print(f"\n── {t('Modules')} ──") + if stats["origin_count"]: + print(f" {t('At the start'):<24}: {stats['origin_count']}") + for version, count, delta in stats["evolution"]: + change = "" if delta is None else f" ({delta:+d})" + print(f" {f'{version}.0':<24}: {count}{change}") + print(f" {t('Removed in total'):<24}: {stats['removed_total']}") + if stats["missing"]: + print(f" {t('Reported missing'):<24}: {len(stats['missing'])}") + if stats["duplicate"]: + print(f" {t('Duplicated'):<24}: {len(stats['duplicate'])}") + + if stats["fixes"]: + print(f"\n── {t('Migration fixes')} ──") + for fix in stats["fixes"]: + mark = "✅" if fix["applied"] else "⬜" + print(f" {mark} {fix['version']}.0 {fix['file']}") + + print(f"\n── {t('COW views')} ──") + if stats["cow"]: + for snap in stats["cow"]: + print( + f" {snap['label']:<18} {str(snap['count']):>4} " + f"{t('views')} {snap['taken_at']}" + ) + else: + print(f" {t('no snapshot')}") + + print(f"\n── {t('Journal')} ──") + print( + f" {len(stats['journal']['commands'])} {t('commands')}, " + f"{len(stats['journal']['comments'])} {t('recorded decisions')}" + ) + + def stats_cow(self, stats, database_name): + """Instantanés COW, et différence entre deux d'entre eux.""" + snaps = stats["cow"] + if len(snaps) < 2: + print(f" {t('Need two snapshots to diff.')}") + return + for index, snap in enumerate(snaps, 1): + print( + f" [{index}] {snap['label']:<18} " + f"{str(snap['count']):>4} {t('views')} {snap['taken_at']}" + ) + raw = input( + f"💬 {t('Diff which two? (e.g. 1,2 — blank to skip)')} : " + ).strip() + parts = [p.strip() for p in raw.replace(",", " ").split()] + if len(parts) != 2 or not all(p.isdigit() for p in parts): + return + first, second = (int(p) - 1 for p in parts) + if not (0 <= first < len(snaps) and 0 <= second < len(snaps)): + return + self.diff_cow_views( + database_name, snaps[first]["label"], snaps[second]["label"] + ) + + @staticmethod + def resume_tui(ctx): + """Resume screen as a TUI. Returns the SAME answer strings as the + prompt, or None when textual is missing (fall back to the prompt).""" + from script.todo import textual_setup + + if not textual_setup.ensure(): + return None + try: + from script.todo.migration_form import run_resume_tui + + return run_resume_tui(ctx) + except ImportError: + return None + + @staticmethod + def version_bumps(dct_progression): + """Odoo versions the step 4 loop walks through, e.g. [13, 14, ..., 18]. + + The per-version lists all end on the target, so the first bump is + « target - len + 1 ». Returns [] when step 4 has not started. + """ + total = max( + [ + len(value) + for key, value in dct_progression.items() + if key.startswith("state_4_") + and key.endswith("_odoo_lst") + and isinstance(value, list) + ] + or [0] + ) + if not total: + return [] + try: + last = int(float(dct_progression["target_odoo_version"])) + except (KeyError, TypeError, ValueError): + return [] + return list(range(last - total + 1, last + 1)) + + @staticmethod + def rewind_version_bump(old_dct_progression, index): + """Replay the step 4 loop from one version bump onwards. + + Only the per-version lists are trimmed from `index`: earlier bumps stay + migrated and steps 0 to 3 are untouched. Resetting the clone entry is + the point — the intermediate database of a failed bump is half + migrated, so it must be dropped and rebuilt from the previous version + rather than upgraded again. + """ + dct_kept = dict(old_dct_progression) + for key, value in old_dct_progression.items(): + if ( + key.startswith("state_4_") + and isinstance(value, list) + and key.endswith(("_odoo_lst", "_module")) + ): + dct_kept[key] = [ + item if i < index else False + for i, item in enumerate(value) + ] + return dct_kept + + @staticmethod + def rewind_progression(old_dct_progression, step): + """Drop the progression of `step` and of every later step. + + Configuration answers (config_*), the zip and the target version are + kept: they are decisions, not progress. Only « state_* » is rewound. + """ + dct_kept = {} + for key, value in old_dct_progression.items(): + if not key.startswith("state_"): + dct_kept[key] = value + continue + index = key[len("state_") :].split("_", 1)[0] + if index.isdigit() and int(index) < step: + dct_kept[key] = value + # The module search fills an in-memory dict the later steps rely on; + # it must run again even when step 0 itself is kept. + dct_kept["state_0_search_missing_module"] = False + print( + f"⏪ {t('Replaying from step')} {step} —" + f" {t(dict(MIGRATION_STEP)[step])}" + ) + return dct_kept + def on_file_selected(self, file_path): self.file_path = file_path todo_file_browser.exit_program() @@ -399,98 +921,30 @@ class TodoUpgrade: self.dct_module_per_dct_version_path = {} default_database_name = "test" + # L'écran de statistiques ne fait rien : on y revient autant de fois + # qu'on veut, et on repose ensuite le choix d'interface. + while True: + ui = self.ask_ui() + if ui is None: + return + if ui != "stats": + break + self.show_stats() + use_tui = ui == "tui" + if os.path.exists(UPGRADE_DATABASE_CONFIG_LOG): - with open(UPGRADE_DATABASE_CONFIG_LOG, "r") as f: - try: - old_dct_progression = json.load(f) - self.dct_progression = { - "migration_file": old_dct_progression.get( - "migration_file" - ), - # More useful to ask this question each time - "target_odoo_version": old_dct_progression.get( - "target_odoo_version" - ), - "date_create": old_dct_progression.get("date_create"), - } - except json.decoder.JSONDecodeError: - print( - f'⚠️ The config file "{UPGRADE_DATABASE_CONFIG_LOG}" is invalid, ignore it.' - ) - - print( - f"✨ Detected migration \"{self.dct_progression.get('migration_file')}\" " - f"to version \"{self.dct_progression.get('target_odoo_version')}\", " - f"please select an option." - ) - - print("[1] Erase progression for a new migration") - print("[2] Reuse database with new process") - print( - "[3] Reuse database without state_4, before looping on next version" - ) - print("[4] Reuse database without configuration") - erase_progression_input = ( - input("💬 Select an option or press to continue : ") - .strip() - .lower() - ) - self.dct_progression = {} - if erase_progression_input in ["2", "3", "4"]: - with open(UPGRADE_DATABASE_CONFIG_LOG, "r") as f: - try: - old_dct_progression = json.load(f) - self.dct_progression = { - "migration_file": old_dct_progression.get( - "migration_file" - ), - # More useful to ask this question each time - # "target_odoo_version": old_dct_progression.get( - # "target_odoo_version" - # ), - "date_create": old_dct_progression.get( - "date_create" - ), - } - for key, value in old_dct_progression.items(): - if erase_progression_input == "3": - if ( - key.startswith("state_0") - or key.startswith("state_1") - or key.startswith("state_2") - or key.startswith("state_3") - ): - self.dct_progression[key] = value - if key.startswith(f"config_state") and not ( - key.startswith(f"config_state_0") - or key.startswith(f"config_state_1") - or key.startswith(f"config_state_2") - or key.startswith(f"config_state_3") - ): - continue - if ( - key.startswith("config_") - and erase_progression_input != "4" - ): - self.dct_progression[key] = value - # Force to search missing module to fill dict - self.dct_progression[ - "state_0_search_missing_module" - ] = False - except json.decoder.JSONDecodeError: - print( - f"⚠️ The config file '{UPGRADE_DATABASE_CONFIG_LOG}' is invalid, ignore it." - ) - - self.write_config() - elif erase_progression_input not in ["1"]: - with open(UPGRADE_DATABASE_CONFIG_LOG, "r") as f: - try: - self.dct_progression = json.load(f) - except json.decoder.JSONDecodeError: - print( - f"⚠️ The config file '{UPGRADE_DATABASE_CONFIG_LOG}' is invalid, ignore it." - ) + old_dct_progression = self.read_progression() + if old_dct_progression: + resumed = self.prompt_resume(old_dct_progression, use_tui) + if resumed is None: + return + self.dct_progression, changed = resumed + if changed: + self.write_config() + elif use_tui: + print(f"ℹ️ {t('No migration in progress to resume.')}") + elif use_tui: + print(f"ℹ️ {t('No migration in progress to resume.')}") if "migration_file" in self.dct_progression: self.file_path = self.dct_progression["migration_file"] @@ -767,6 +1221,31 @@ class TodoUpgrade: self.write_config() print("✅ -> Restore database") + already_update_state_1 = False + + if not self.dct_progression.get("state_1_update_all"): + print( + "[1] Update all addons before neutralize (already neutralize by odoo if supported)" + ) + wait_continue = ( + input( + "💬 Do you need to upgrade before a database neutralization, press to ignore : " + ) + .strip() + .lower() + ) + if wait_continue == "1": + status, cmd_executed = self.todo_upgrade_execute( + f"./script/addons/update_addons_all.sh {database_name}", + single_source_odoo=True, + ) + + if not status: + already_update_state_1 = True + self.dct_progression["state_1_update_all"] = True + self.write_config() + + print("✅ -> Update database before neutralize by module") print("✅ -> Neutralize database") if do_neutralize: @@ -784,8 +1263,9 @@ class TodoUpgrade: # ./script/database/migrate/process_backup_file.py --path_backup_zip image_db/db.zip --path_output_zip image_db/dbFIX.zip --word_to_delete discuss_channel_channel_type_not_null # Puis faire un retry de la commande, sinon rien - self.dct_progression["state_1_neutralize_database"] = True - self.write_config() + # Only record the step when it actually succeeded. The previous + # unconditional assignment made the test below dead code: a failed + # neutralization was remembered as done and skipped on resume. if not status: self.dct_progression["state_1_neutralize_database"] = True self.write_config() @@ -801,18 +1281,12 @@ class TodoUpgrade: ) if not is_state_4_reach_open_upgrade: - lst_module_to_uninstall = [] - uninstall_module_list_file = os.path.join( - "script", - "odoo", - "migration", - f"uninstall_module_list_odoo{start_version * 10}_to_odoo{(start_version + 1) * 10}.txt", + lst_module_to_uninstall, lst_uninstall_reason = ( + self.read_uninstall_module_list(start_version, database_name) ) - if os.path.exists(uninstall_module_list_file): - with open(uninstall_module_list_file, "r") as f: - lst_module_to_uninstall = [ - a.strip() for a in f.readline().split() - ] + if lst_uninstall_reason: + print("✨ Modules to uninstall before migration :") + self.print_uninstall_reason(lst_uninstall_reason) if config_state_1_uninstall_module: lst_module_to_uninstall = ( @@ -856,7 +1330,10 @@ class TodoUpgrade: print(f"🔷 {msg}") self.add_comment_progression(msg) - if not self.dct_progression.get("state_2_update_all"): + if ( + not self.dct_progression.get("state_2_update_all") + and not already_update_state_1 + ): status, cmd_executed = self.todo_upgrade_execute( f"./script/addons/update_addons_all.sh {database_name}", single_source_odoo=True, @@ -865,6 +1342,20 @@ class TodoUpgrade: self.dct_progression["state_2_update_all"] = True self.write_config() + # Predict the website COW views that the NEXT version bump will break. + # A copy-on-write view freezes the structure of the module view it was + # copied from; when that module view changes mode between two versions, + # the copy keeps an arch written for the old mode and the upgrade dies + # on « Element ... cannot be located in parent view ». Reporting it here + # -- before hours of migration -- leaves time to arbitrate. + # Only the next bump can be predicted: the modes in database describe + # the current version. + self.todo_upgrade_execute( + f"{PYTHON_BIN} ./script/odoo/migration/check_cow_views.py" + f" -d {database_name} -t odoo{start_version + 1}.0", + wait_at_error=False, + ) + msg = "3 - Clean up database before data migration" print(f"🔷 {msg}") self.add_comment_progression(msg) @@ -1003,7 +1494,20 @@ class TodoUpgrade: # Duplicate database cmd_clone_database = f"./odoo_bin.sh db --clone --from_database {last_database_name} --database {database_name_upgrade}" - self.todo_upgrade_execute(cmd_clone_database) + status, cmd_executed = self.todo_upgrade_execute( + cmd_clone_database + ) + + # Everything downstream runs against this clone: if it failed, + # do not mark it done (a rerun would skip the clone and migrate + # a missing or truncated database). + if status: + print( + f"❌ -> Clone to Odoo{next_version} FAILED (status" + f" {status}). Stopping: '{database_name_upgrade}' is" + " not usable." + ) + return lst_clone_odoo[index] = True self.dct_progression["state_4_clone_odoo_lst"] = lst_clone_odoo @@ -1022,9 +1526,24 @@ class TodoUpgrade: ) if not lst_module_uninstall_module[index]: - lst_module_to_uninstall = config_state_4_uninstall_module[ - index - ] + lst_module_to_uninstall = ( + config_state_4_uninstall_module[index] or [] + ) + # Same file convention as step 1, one file per version bump: + # uninstall_module_list_odoo130_to_odoo140.txt is read HERE, + # right before the 13 -> 14 data migration. Without this the + # per-bump files existed in name only and were never read. + lst_file, lst_detail = self.read_uninstall_module_list( + next_version - 1, database_name + ) + if lst_detail: + print( + f"✨ Modules to uninstall before Odoo{next_version} :" + ) + self.print_uninstall_reason(lst_detail) + lst_module_to_uninstall = list( + dict.fromkeys(list(lst_module_to_uninstall) + lst_file) + ) if lst_module_to_uninstall: self.uninstall_from_database( @@ -1475,18 +1994,47 @@ class TodoUpgrade: if not lst_fix_migration_odoo[index]: print("") - file_path_fix_migration = os.path.join( - "script", - "odoo", - "migration", - f"fix_migration_odoo{(next_version-1)*10}_to_odoo{next_version*10}.py", + stem = os.path.join( + PATH_MIGRATION_GLOBAL, + f"fix_migration_odoo{(next_version - 1) * 10}" + f"_to_odoo{next_version * 10}", ) - if os.path.exists(file_path_fix_migration): - self.todo_upgrade_execute( - f"cat ./{file_path_fix_migration} | ./odoo{next_version}.0/odoo/odoo-bin shell -d {database_name_upgrade}", + # Two flavours. « .sql » runs through psql: no Odoo registry, + # so it works on a database not yet migrated -- exactly when + # loading it with the TARGET version's code would fail. « .py » + # is piped into the Odoo shell when the ORM is really needed. + file_path_fix_migration = "" + cmd_fix_migration = "" + if os.path.exists(f"{stem}.sql"): + file_path_fix_migration = f"{stem}.sql" + cmd_fix_migration = ( + f"psql -v ON_ERROR_STOP=1 -d {database_name_upgrade}" + f" -f ./{file_path_fix_migration}" + ) + elif os.path.exists(f"{stem}.py"): + file_path_fix_migration = f"{stem}.py" + cmd_fix_migration = ( + f"cat ./{file_path_fix_migration} |" + f" ./odoo{next_version}.0/odoo/odoo-bin shell" + f" -d {database_name_upgrade}" + ) + if file_path_fix_migration: + status, cmd_executed = self.todo_upgrade_execute( + cmd_fix_migration, single_source_odoo=True, ) + # A fix that did not run must not be recorded as applied, + # otherwise the rerun skips it and OpenUpgrade hits the very + # problem the fix exists to prevent. + if status: + print( + f"❌ -> Fix migration Odoo{next_version} FAILED" + f" (status {status}):" + f" {file_path_fix_migration}" + ) + return + lst_fix_migration_odoo[index] = file_path_fix_migration self.dct_progression["state_4_fix_migration_odoo_lst"] = ( lst_fix_migration_odoo @@ -1545,13 +2093,68 @@ class TodoUpgrade: cmd_upgrade = f"./run.sh --upgrade-path=./odoo{next_version}.0/OCA_OpenUpgrade/openupgrade_scripts/scripts --update all -c config.conf --stop-after-init --no-http --load=base,web,openupgrade_framework -d {database_name_upgrade}" lst_upgrade_odoo[index] = cmd_upgrade - self.todo_upgrade_execute( + # Record the website COW views before the data migration. The + # upgrade silently deletes and recreates copies (measured on + # 12->13: 16 copies dropped, 13 created, children re-parented), + # and rewrites the arch of many others. Without a before/after + # record, "the site looks wrong" cannot be investigated. + self.snapshot_cow_views( + database_name_upgrade, f"before_{next_version}" + ) + + # Take the copies that cannot survive this bump out of the way, + # otherwise the data migration dies on them. Offered, not + # forced: their arch is a real customization. + self.neutralize_cow_views(database_name_upgrade, next_version) + + # wait_at_error=False on purpose: the generic « [1] to redo the + # command » would replay OpenUpgrade on a database it has just + # half migrated, which never recovers. The failure is handled + # below by dropping the clone flag so the replay REBUILDS the + # intermediate database from the previous version. + status, cmd_executed = self.todo_upgrade_execute( cmd_upgrade, new_env={ "OPENUPGRADE_TARGET_VERSION": f"{next_version}.0" }, + wait_at_error=False, + ) + + # This is THE data migration. Recording it as done when it + # failed used to send the loop to the next version on top of a + # half-migrated database. Stop here instead: the state stays + # unset, so a rerun replays this version. + if status: + # The intermediate database is now half migrated and must + # not be reused: drop the clone flag so the rerun rebuilds + # it from the pristine source. Without this the replay + # would restart OpenUpgrade on top of the broken clone. + lst_clone_odoo[index] = False + self.dct_progression["state_4_clone_odoo_lst"] = ( + lst_clone_odoo + ) + self.write_config() + print( + f"\n❌ -> Database migration to Odoo{next_version}" + f" FAILED (status {status}).\n" + f" '{database_name_upgrade}' is now half migrated:" + " replaying the command on it would never recover, so" + " it is NOT offered.\n" + " The clone step has been reset. Fix the cause, then" + " relaunch the migration and answer [c] (continue):" + f" '{database_name_upgrade}' will be dropped and" + " rebuilt from the previous version before retrying." + ) + return + + self.snapshot_cow_views( + database_name_upgrade, f"after_{next_version}" + ) + self.diff_cow_views( + database_name_upgrade, + f"before_{next_version}", + f"after_{next_version}", ) - # TODO detect error self.dct_progression["state_4_upgrade_odoo_lst"] = ( lst_upgrade_odoo @@ -1738,18 +2341,217 @@ class TodoUpgrade: } return dct_module + def snapshot_cow_views(self, database_name, label): + """Record the website COW views of a database under private/. + + Never blocks the migration: a snapshot is forensic material, its + absence must not stop an upgrade. + """ + self.todo_upgrade_execute( + f"{PYTHON_BIN} ./script/odoo/migration/snapshot_cow_views.py" + f" -d {database_name} -l {label}", + wait_at_error=False, + ) + + def neutralize_cow_views(self, database_name, next_version): + """Offer to neutralize the COW views that would break this bump. + + Renaming their key unpairs them from the module view, so the upgrade + stops choking on them. Nothing is deleted and the operation is + reversible (neutralize_cow_views.py --restore), but the choice belongs + to the user: those copies carry real customizations. + """ + cmd = ( + f"{PYTHON_BIN} ./script/odoo/migration/neutralize_cow_views.py" + f" -d {database_name} -t odoo{next_version}.0" + ) + status, cmd_executed, output = self.todo_upgrade_execute( + cmd, get_output=True, wait_at_error=False + ) + if "No website COW view to neutralize" in "\n".join(output or []): + return + + answer = ( + input( + "💬 Neutralize these copies so the upgrade can proceed?" + " Their arch is kept and the change is reversible." + " (Y/n) : " + ) + .strip() + .lower() + ) + if answer == "n": + print( + "⚠️ -> Skipped. The data migration will very likely stop on" + " these views." + ) + return + self.todo_upgrade_execute(f"{cmd} --apply", wait_at_error=False) + + def diff_cow_views(self, database_name, label_before, label_after): + """Print what the version bump did to the website COW views.""" + directory = os.path.join( + PATH_MIGRATION_PRIVATE, database_name, "cow_snapshots" + ) + path_before = os.path.join(directory, f"{label_before}.json") + path_after = os.path.join(directory, f"{label_after}.json") + if not (os.path.exists(path_before) and os.path.exists(path_after)): + return + self.todo_upgrade_execute( + f"{PYTHON_BIN} ./script/odoo/migration/snapshot_cow_views.py" + f" --diff {path_before} {path_after}", + wait_at_error=False, + ) + + @staticmethod + def parse_module_list_file(file_path): + """Read a module list file, return [(module, reason), ...]. + + Accepted syntax, one module per line with an optional justification: + + queue_job # blocks 12->13, trigger queue_job_notify + mgmtsystem_hazard # not ported to 13.0 + + Commas and several names per line are also accepted, so a list copied + from a command line works as-is. Blank lines and full-line comments are + ignored. + + The previous parser was « f.readline().split() »: it kept only the FIRST + line, so a multi-line list was silently truncated, and a comma-separated + list collapsed into one bogus module name. + """ + lst_module = [] + with open(file_path, "r", encoding="utf-8") as f: + for line in f: + content, _, reason = line.partition("#") + for module_name in content.replace(",", " ").split(): + lst_module.append((module_name, reason.strip())) + return lst_module + + @staticmethod + def print_uninstall_reason(lst_detail): + """Show WHY each module goes away. + + Which modules must be dropped depends on the database, and a removal + without a stated reason is a decision nobody can review later. + """ + for name, reason, origin in lst_detail: + print( + f" - {name}" + + (f" — {reason}" if reason else " — ⚠️ no reason given") + + f" [{origin}]" + ) + + def read_uninstall_module_list(self, start_version, database_name): + """Modules to uninstall before migrating start_version -> next. + + Reads the per-database private list first, then the shared versioned + defaults; duplicates are dropped, keeping the first occurrence. + + Returns (lst_module, lst_detail) where lst_detail carries + (module, reason, origin_file) so the caller can justify each removal. + """ + file_name = ( + f"uninstall_module_list_odoo{start_version * 10}" + f"_to_odoo{(start_version + 1) * 10}.txt" + ) + lst_path = [ + os.path.join(PATH_MIGRATION_PRIVATE, database_name, file_name), + os.path.join(PATH_MIGRATION_GLOBAL, file_name), + ] + + lst_module = [] + lst_detail = [] + for file_path in lst_path: + if not os.path.exists(file_path): + continue + for module_name, reason in self.parse_module_list_file(file_path): + if module_name in lst_module: + continue + lst_module.append(module_name) + lst_detail.append((module_name, reason, file_path)) + return lst_module, lst_detail + + def split_present_missing(self, lst_module): + """Split a module list into (present, missing) against the ACTIVE code. + + « Missing » means the addons path no longer holds the module — not + that it is absent from the database. The distinction matters because + check_addons_exist.py refuses the WHOLE uninstall for a single missing + name, so the modules that ARE there never get uninstalled either. + """ + lst_missing, _lst_duplicate = self.check_addons_exist(lst_module) + set_missing = set(lst_missing or []) + return ( + [name for name in lst_module if name not in set_missing], + [name for name in lst_module if name in set_missing], + ) + + def prompt_uninstall_missing(self, lst_present, lst_missing): + """Ask what to do when part of the list has no code left. + Returns the list to actually uninstall (possibly empty).""" + print() + print( + f"⚠️ {len(lst_missing)} " + f"{t('modules of the list have no code in the active Odoo:')}" + ) + for name in lst_missing: + print(f" {name}") + print(f" {t('Odoo cannot uninstall a module whose code is gone;')}") + print(f" {t('one of them fails the whole uninstall command.')}") + print() + if lst_present: + print( + f" {len(lst_present)} " + f"{t('are present and can be uninstalled:')}" + ) + for name in lst_present: + print(f" {name}") + else: + print(f" {t('No module of the list is present.')}") + print() + if lst_present: + print( + f" [1] {t('Uninstall the present ones, skip the missing')}" + " *" + ) + print(f" [2] {t('Try the whole list anyway (it will fail)')}") + print(f" [3] {t('Uninstall nothing, continue')}") + answer = input(f"💬 {t('Your choice')} : ").strip() + if answer == "2": + return lst_present + lst_missing + if answer == "3" or not lst_present: + return [] + return lst_present + def uninstall_from_database( self, lst_module_to_uninstall, database_name, actual_version ): if not lst_module_to_uninstall: return + # Sort out what the active code still holds BEFORE calling the script: + # it aborts on the first missing name and takes the rest down with it. + lst_present, lst_missing = self.split_present_missing( + lst_module_to_uninstall + ) + if lst_missing: + self.add_comment_progression( + "uninstall - no code for: " + ", ".join(lst_missing) + ) + lst_module_to_uninstall = self.prompt_uninstall_missing( + lst_present, lst_missing + ) + if not lst_module_to_uninstall: + print(f"⏭ {t('Nothing uninstalled.')}") + return uninstall_module = ",".join(lst_module_to_uninstall) self.todo_upgrade_execute( f"./script/addons/uninstall_addons.sh {database_name} {uninstall_module}", single_source_odoo=True, ) - # Update list installed module + # Update list installed module — only what was REALLY uninstalled, so + # a module left in place stays counted as installed. self.dct_module_per_version[actual_version] = sorted( list( set(self.dct_module_per_version[actual_version]) @@ -1894,18 +2696,38 @@ class TodoUpgrade: self.lst_command_executed.append(cmd_executed) self.dct_progression["command_executed"] = self.lst_command_executed self.write_config() - if status and wait_at_error: - print("[1] to redo the command") - wait_status = ( - input( - "💬 Error detected, press to continue or ctrl+c to stop : " + # None means « the command never reported a status » -> treat it as a + # failure, never as a success (defence in depth: exec_command_live now + # always sets one, but a silent None must not skip this prompt). + if (status is None or status) and wait_at_error: + database_name = self.database_from_command(cmd) + while True: + print("[1] to redo the command") + if database_name: + print( + f"[2] {t('Check the COW views that drifted')}" + f" ({database_name})" + ) + wait_status = ( + input( + "💬 Error detected, press to continue or ctrl+c to" + " stop : " + ) + .strip() + .lower() ) - .strip() - .lower() - ) - # psycopg2.errors.UndefinedTable: relation "discuss_channel" does not exist - # LIGNE 1 : SELECT "discuss_channel"."id" FROM "discuss_channel" WHERE (... + # psycopg2.errors.UndefinedTable: relation "discuss_channel" does not exist + # LIGNE 1 : SELECT "discuss_channel"."id" FROM "discuss_channel" WHERE (... + + if wait_status == "2" and database_name: + # Le motif d'échec le plus fréquent ici est « Element + # cannot be located in parent view » : une copie + # COW en retard sur sa vue module. On propose l'outil sur + # place, puis on repose le choix pour rejouer. + self.check_stale_cow_views(database_name) + continue + break if wait_status == "1": return self.todo_upgrade_execute(