diff --git a/script/odoo/migration/cow_drift.py b/script/odoo/migration/cow_drift.py new file mode 100644 index 0000000..a8c5e85 --- /dev/null +++ b/script/odoo/migration/cow_drift.py @@ -0,0 +1,342 @@ +#!/usr/bin/env python3 +# © 2021-2026 TechnoLibre (http://www.technolibre.ca) +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl) + +"""Show what a website COW copy holds, and why the next version breaks it. + +Read-only. Answers the two questions someone asks before neutralizing a copy, +and neither is answerable from the warning alone: + +**What do I lose?** The copy is compared with the module view it shadows. That +is the customization someone made, and often it is three lines — an id, a +container width — for which nobody would hold up a migration. + +**Why does it break?** The module declaration is shown in the current version +and in the target. The pair is the whole explanation: a template declared +without ``inherit_id`` is a standalone document, one declared with it must +hold inheritance specs. A copy frozen in the first shape cannot be applied in +the second, and Odoo stops on « cannot be located in parent view ». + +Nothing here writes. Neutralizing is ``neutralize_cow_views.py --apply``, and +undoing it is ``--restore``. +""" + +import argparse +import difflib +import json +import os +import re +import subprocess +import sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +from check_cow_views import analyse, find_module_dir # noqa: E402 + +# The declaration of a template spans a few lines; showing the opening tag and +# what follows is enough to see its shape, and short enough to compare two +# versions side by side without scrolling. +DECL_LINES = 4 + + +def run_psql(database, sql): + """Run a statement and return stdout, raising on failure.""" + result = subprocess.run( + ["psql", "-X", "-w", "-d", database, "-tAc", sql], + capture_output=True, + text=True, + ) + if result.returncode: + raise RuntimeError( + f"Query failed on '{database}': {result.stderr.strip()}" + ) + return result.stdout + + +def unwrap_arch(value): + """The arch as text, whatever the column type. + + ``arch_db`` is text up to 15.0 and jsonb from 16.0. The same unwrapping as + the sibling scripts, kept identical on purpose: a second reading of the + same column that differs on an edge case would report a drift nobody made. + """ + text = (value or "").strip() + if text.startswith("{") and '"' in text: + try: + data = json.loads(text) + except ValueError: + return value or "" + if isinstance(data, dict) and data: + for lang in ("en_US", *sorted(data)): + if isinstance(data.get(lang), str): + return data[lang] + return value or "" + + +def fetch_arch(database, view_id): + """The stored arch of one view.""" + return unwrap_arch( + run_psql( + database, + f"SELECT arch_db::text FROM ir_ui_view WHERE id = {int(view_id)};", + ) + ) + + +def fetch_module_view(database, key): + """(id, arch) of the module view a copy shadows, or (None, ''). + + The module view is the one carrying that key WITHOUT a website: that is + exactly the pairing Odoo itself makes, and the reason renaming a key is + enough to unpair a copy. + """ + safe = key.replace("'", "''") + out = run_psql( + database, + "SELECT id, arch_db::text FROM ir_ui_view" + f" WHERE key = '{safe}' AND website_id IS NULL" + " ORDER BY id LIMIT 1;", + ).strip() + if not out: + return None, "" + view_id, _, arch = out.partition("|") + return int(view_id), unwrap_arch(arch) + + +def declaration(version_dir, key): + """(path, snippet) of how the sources of one version declare this key. + + Returns None when the module or the node is not found — a fact worth + showing as such, since « the module no longer declares it » is itself a + reason a copy breaks. + """ + module_name, _, template_id = key.partition(".") + if not template_id: + return None + module_dir = find_module_dir(version_dir, module_name) + if module_dir is None: + return None + try: + from lxml import etree + except ImportError: + return None + + pattern = os.path.join(module_dir, "**", "*.xml") + import glob + + for path in sorted(glob.glob(pattern, recursive=True)): + try: + tree = etree.parse(path) + except etree.XMLSyntaxError: + continue + for element in tree.getroot().iter(): + if element.get("id") != template_id: + continue + if element.tag not in ("template", "record"): + continue + # The source line, not a re-serialization: what the file actually + # says is what a reader will grep for. + with open(path, "r", encoding="utf-8", errors="replace") as handle: + lines = handle.read().splitlines() + start = max((element.sourceline or 1) - 1, 0) + return path, "\n".join(lines[start : start + DECL_LINES]) + return None + + +def database_version_dir(database): + """The odoo.0 directory matching what the DATABASE says it is. + + Not `.odoo-version`: the checkout is switched to the TARGET before this + runs, so reading it would compare the target with itself and show the same + declaration twice — which is what it did until this was measured on a real + migration. The database, at that moment, is still on the previous version + and says so in ir_module_module. + """ + try: + out = run_psql( + database, + "SELECT latest_version FROM ir_module_module" + " WHERE name = 'base';", + ).strip() + except RuntimeError: + return None + if not out: + return None + parts = out.split(".") + if len(parts) < 2: + return None + return f"odoo{parts[0]}.{parts[1]}" + + +def collect(database, target_version, current_version=None): + """Everything needed to judge each at-risk copy. No writes.""" + current_version = current_version or database_version_dir(database) + lst_at_risk, _, _ = analyse(database, target_version) + lst_finding = [] + for view_id, key, mode, target_mode, website_id, reason in lst_at_risk: + module_id, module_arch = fetch_module_view(database, key) + lst_finding.append( + { + "id": view_id, + "key": key, + "mode": mode, + "target_mode": target_mode, + "website_id": website_id, + "reason": reason, + "copy_arch": fetch_arch(database, view_id), + "module_id": module_id, + "module_arch": module_arch, + "decl_current": ( + declaration(current_version, key) + if current_version + else None + ), + "decl_target": declaration(target_version, key), + "current_version": current_version, + "target_version": target_version, + } + ) + return lst_finding + + +def render_diff(finding): + """What the copy changed, compared with the module view it shadows.""" + lines = [ + f"── id={finding['id']} {finding['key']}" + f" (website={finding['website_id']}) ──", + "", + ] + if finding["module_id"] is None: + lines += [ + " No module view carries this key, so there is nothing to compare", + " against: this copy is a page made in the website editor.", + ] + return "\n".join(lines) + + left = finding["module_arch"].splitlines() + right = finding["copy_arch"].splitlines() + diff = [ + line + for line in difflib.unified_diff( + left, + right, + fromfile=f"module id={finding['module_id']}", + tofile=f"copy id={finding['id']}", + lineterm="", + n=1, + ) + ] + if len(diff) <= 2: + lines.append(" The copy is identical to the module view.") + return "\n".join(lines) + lines += [f" {line}" for line in diff] + n_plus = sum( + 1 for x in diff if x.startswith("+") and not x.startswith("+++") + ) + n_minus = sum( + 1 for x in diff if x.startswith("-") and not x.startswith("---") + ) + lines += [ + "", + f" {n_plus} line(s) added, {n_minus} removed — this is what" + " neutralizing gives up.", + ] + return "\n".join(lines) + + +def render_shape(finding): + """Why it breaks: the declaration in each version, side by side.""" + lines = [ + f"── id={finding['id']} {finding['key']} ──", + "", + f" {finding['reason']}", + "", + ] + for label, decl in ( + (finding["current_version"], finding["decl_current"]), + (finding["target_version"], finding["decl_target"]), + ): + lines.append(f" ") + if decl is None: + lines += [ + " (the module no longer declares this template)", + "", + ] + continue + path, snippet = decl + lines.append(f" ") + lines += [f" {line}" for line in snippet.splitlines()] + shape = ( + "inheritance specs (needs inherit_id)" + if re.search(r"inherit_id\s*=", snippet) + else "a standalone template" + ) + lines += [f" -> {shape}", ""] + lines += [ + " A copy frozen in one shape cannot be applied in the other: Odoo", + " stops on « cannot be located in parent view ».", + ] + return "\n".join(lines) + + +def render_all(lst_finding, shape=False): + """The whole report, one block per finding.""" + if not lst_finding: + return "✅ No website COW view is at risk.\n" + render = render_shape if shape else render_diff + return "\n\n".join(render(f) for f in lst_finding) + "\n" + + +def main(argv=None): + parser = argparse.ArgumentParser( + description=( + "Show what each at-risk website COW copy holds, and why the next" + " version breaks it (read-only)." + ) + ) + parser.add_argument("-d", "--database", required=True) + parser.add_argument( + "-t", + "--target_version", + required=True, + help="target Odoo source directory, e.g. odoo13.0", + ) + parser.add_argument( + "--current", + default=None, + help="current Odoo source directory (default: from .odoo-version)", + ) + parser.add_argument( + "--shape", + action="store_true", + help="show the declarations instead of the customization diff", + ) + parser.add_argument( + "--tui", + action="store_true", + help="browse full screen, switching between the two views", + ) + config = parser.parse_args(argv) + + if not os.path.isdir(config.target_version): + print(f"❌ Target version '{config.target_version}' not found.") + return 2 + try: + lst_finding = collect( + config.database, config.target_version, config.current + ) + except RuntimeError as exc: + print(f"❌ {exc}") + return 2 + + if config.tui and lst_finding: + from cow_drift_tui import run_tui + + if run_tui(lst_finding): + return 1 + print(render_all(lst_finding, shape=config.shape)) + return 1 if lst_finding else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/script/odoo/migration/cow_drift_tui.py b/script/odoo/migration/cow_drift_tui.py new file mode 100644 index 0000000..953ef65 --- /dev/null +++ b/script/odoo/migration/cow_drift_tui.py @@ -0,0 +1,162 @@ +#!/usr/bin/env python3 +# © 2021-2026 TechnoLibre (http://www.technolibre.ca) +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl) + +"""Full-screen browsing of the at-risk website COW copies. + +Read-only, without exception: this screen exists to decide, and deciding +supposes having looked. Neutralizing stays a separate, explicit command. + +Two views of the same copy, one key apart +----------------------------------------- +« What do I lose » and « why does it break » are different questions with +different answers, and putting them side by side would halve the width of +each on a screen that already shows XML. They share the pane instead, and +``space`` switches — the header always says which one is showing, because a +diff and a declaration look alike at a glance. +""" + +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +CSS = """ +Screen { layout: vertical; } +#head { height: 3; padding: 0 1; background: $panel; color: $text; } +#body { height: 1fr; } +#views { width: 38; border-right: solid $accent; } +#pane { width: 1fr; padding: 0 1; } +""" + + +def build_app(lst_finding): + """Build the application. Textual is imported here, not at module level. + + The module stays importable — and therefore testable — on a machine + without Textual, which is also what lets the caller fall back to the text + report rather than fail. + """ + from textual.app import App, ComposeResult + from textual.containers import Horizontal, VerticalScroll + from textual.widgets import DataTable, Footer, Header, Static + + from cow_drift import render_diff, render_shape + + class DriftApp(App): + CSS = globals()["CSS"] + BINDINGS = [ + ("q,escape", "quit", "Quit"), + ("space,tab", "toggle", "Diff / declarations"), + ("c", "copy", "Copy"), + ] + + def __init__(self, lst_finding): + super().__init__() + self.lst_finding = lst_finding + self.shape = False + self.index = 0 + + def compose(self) -> ComposeResult: + yield Header() + yield Static("", id="head") + with Horizontal(id="body"): + yield DataTable(id="views", cursor_type="row") + with VerticalScroll(id="pane"): + yield Static("", id="content") + yield Footer() + + def on_mount(self): + self.title = "Website COW copies at risk" + table = self.query_one("#views", DataTable) + table.add_columns("copy", "+/-") + for finding in self.lst_finding: + table.add_row( + (finding["key"] or str(finding["id"]))[:28], + self._weight(finding), + key=str(finding["id"]), + ) + self._show() + + def _weight(self, finding): + """How much the copy diverges, so the list can be triaged.""" + if finding["module_id"] is None: + return "—" + import difflib + + diff = list( + difflib.unified_diff( + finding["module_arch"].splitlines(), + finding["copy_arch"].splitlines(), + lineterm="", + n=0, + ) + ) + plus = sum( + 1 + for x in diff + if x.startswith("+") and not x.startswith("+++") + ) + minus = sum( + 1 + for x in diff + if x.startswith("-") and not x.startswith("---") + ) + return f"+{plus}/-{minus}" + + def _show(self): + if not self.lst_finding: + return + finding = self.lst_finding[self.index] + which = "declarations" if self.shape else "what the copy changed" + self.query_one("#head", Static).update( + f"{finding['key']} · id={finding['id']}" + f" · website={finding['website_id']}\n" + f"[{which}] — space to switch" + ) + render = render_shape if self.shape else render_diff + self.query_one("#content", Static).update(render(finding)) + + def on_data_table_row_highlighted(self, event): + if event.data_table.id == "views" and self.lst_finding: + self.index = event.cursor_row + self._show() + + def action_toggle(self): + self.shape = not self.shape + self._show() + + def action_copy(self): + if not self.lst_finding: + return + from cow_drift import render_diff, render_shape + + render = render_shape if self.shape else render_diff + # Truncated keeping the END: that is where the added lines are, + # and what someone is most often after. + self.copy_to_clipboard( + render(self.lst_finding[self.index])[-100_000:] + ) + self.notify("Copied.") + + return DriftApp(lst_finding) + + +def run_tui(lst_finding, run_app=True): + """Open the screen. False when it could not be shown. + + False is not a failure: the caller prints the text report instead, which + is the same information without the navigation. + """ + if not lst_finding: + return False + if not sys.stdout.isatty(): + return False + try: + app = build_app(lst_finding) + except ImportError: + return False + if not run_app: + return app + app.run() + return True diff --git a/script/odoo/migration/neutralize_cow_views.py b/script/odoo/migration/neutralize_cow_views.py index 7e32425..021cef8 100755 --- a/script/odoo/migration/neutralize_cow_views.py +++ b/script/odoo/migration/neutralize_cow_views.py @@ -40,7 +40,7 @@ DEFAULT_PREFIX = "zz_cow_archive" def run_psql(database, sql): """Run a statement and return stdout, raising on failure.""" result = subprocess.run( - ["psql", "-d", database, "-tAc", sql], + ["psql", "-X", "-w", "-d", database, "-tAF", "|", "-c", sql], capture_output=True, text=True, ) @@ -66,6 +66,39 @@ def neutralize(database, lst_view_id, prefix): return int(output or 0) +def list_archived(database, prefix): + """The copies a previous neutralization put aside. + + Knowing what was archived is the other half of --restore: a key renamed + months ago is invisible in the interface — the copy is inactive and no + longer pairs with anything — so without this listing the only trace is + someone's memory of having run --apply. + """ + output = run_psql( + database, + "SELECT id, substring(key from " + str(len(prefix) + 2) + ")," + " COALESCE(website_id::text, ''), active," + " octet_length(arch_db::text)" + f" FROM ir_ui_view WHERE key LIKE '{prefix}.%' ORDER BY id;", + ) + lst_row = [] + for line in output.splitlines(): + if not line.strip(): + continue + parts = line.split("|") + if len(parts) >= 5: + lst_row.append( + { + "id": int(parts[0]), + "key": parts[1], + "website_id": parts[2] or None, + "active": parts[3] == "t", + "arch_bytes": int(parts[4] or 0), + } + ) + return lst_row + + def restore(database, prefix): """Undo a neutralization: strip the prefix and reactivate.""" output = run_psql( @@ -105,8 +138,43 @@ def main(): action="store_true", help="undo a previous neutralization and reactivate the copies", ) + parser.add_argument( + "--list", + dest="list_archived", + action="store_true", + help="list the copies a previous neutralization put aside", + ) config = parser.parse_args() + try: + return _run(config, parser) + except RuntimeError as exc: + # Une base absente ou un PostgreSQL arrêté est une erreur d'usage, pas + # un défaut de l'outil : une trace d'appel ferait chercher le bogue au + # mauvais endroit. + print(f"❌ {exc}") + return 2 + + +def _run(config, parser): + if config.list_archived: + lst_row = list_archived(config.database, config.prefix) + if not lst_row: + print(f"✅ -> No '{config.prefix}.' view on '{config.database}'.") + return 0 + print(f"ℹ {len(lst_row)} archived COW view(s) on '{config.database}':") + for row in lst_row: + state = "active" if row["active"] else "inactive" + print( + f" - id={row['id']} website={row['website_id'] or '-'}" + f" {row['key']} ({state}, {row['arch_bytes']} B)" + ) + print( + " Their arch is intact. Restore them all with --restore, once" + " the module view they shadow has the shape they expect." + ) + return 0 + if config.restore: count = restore(config.database, config.prefix) print(f"✅ -> {count} COW view(s) restored on '{config.database}'.") diff --git a/script/todo/todo_upgrade.py b/script/todo/todo_upgrade.py index 6c37836..f4ade39 100755 --- a/script/todo/todo_upgrade.py +++ b/script/todo/todo_upgrade.py @@ -2371,15 +2371,35 @@ class TodoUpgrade: if "No website COW view to neutralize" in "\n".join(output or []): return - answer = ( - input( - "💬 Neutralize these copies so the upgrade can proceed?" - " Their arch is kept and the change is reversible." - " (Y/n) : " - ) - .strip() - .lower() + # « v » et « w » avant de répondre : la question demande de renoncer à + # une personnalisation sans avoir montré laquelle. Souvent trois lignes + # — un id, une largeur de conteneur — mais parfois une page entière, et + # rien dans l'avertissement ne permet de les distinguer. + show = ( + f"{PYTHON_BIN} ./script/odoo/migration/cow_drift.py" + f" -d {database_name} -t odoo{next_version}.0" ) + while True: + answer = ( + input( + "💬 Neutralize these copies so the upgrade can proceed?" + " Their arch is kept and the change is reversible." + " (Y/n, v = view the differences, w = full screen) : " + ) + .strip() + .lower() + ) + if answer == "v": + self.todo_upgrade_execute(show, wait_at_error=False) + self.todo_upgrade_execute( + f"{show} --shape", wait_at_error=False + ) + continue + if answer == "w": + self.todo_upgrade_execute(f"{show} --tui", wait_at_error=False) + continue + break + if answer == "n": print( "⚠️ -> Skipped. The data migration will very likely stop on" @@ -2387,6 +2407,11 @@ class TodoUpgrade: ) return self.todo_upgrade_execute(f"{cmd} --apply", wait_at_error=False) + print( + "ℹ -> List them later with:" + f" {PYTHON_BIN} ./script/odoo/migration/neutralize_cow_views.py" + f" -d {database_name} --list" + ) def diff_cow_views(self, database_name, label_before, label_after): """Print what the version bump did to the website COW views."""