diff --git a/script/odoo/migration/check_stale_scss.py b/script/odoo/migration/check_stale_scss.py new file mode 100755 index 0000000..e9c51e3 --- /dev/null +++ b/script/odoo/migration/check_stale_scss.py @@ -0,0 +1,326 @@ +#!/usr/bin/env python3 +# © 2021-2026 TechnoLibre (http://www.technolibre.ca) +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl) + +"""Predict which customized SCSS will break the next version bump. + +Background +---------- +Customizing a website writes SCSS into ``ir_attachment``: files whose URL +carries ``.custom.``. That copy is frozen the day it is written, and it keeps +using the variables of THAT version. + +A module can rename its variables between two Odoo versions. Measured on a +real database: ``website/static/src/scss/primary_variables.scss`` declared +``$o-theme-font-number`` in 12.0 and replaced the whole mechanism in 13.0 with +a ``$o-website-values-palettes`` map. A customization written in 2020 still +asked for the old name. The bundle then stops on:: + + Error: Undefined variable: "$o-theme-font-number". + This error occured while compiling the bundle 'web.assets_frontend' + +So the rule is the same as for COW views: + + a customization breaks when what it USES stops being DEFINED between + version N and version N+1. + +Why not just start Odoo and open a page +--------------------------------------- +Because that only tells you *after* the bump, on a page that renders a +« Style error » with no name attached: not which attachment, not which +variable, not since when. And a page has to be reached at all — a broken +frontend bundle is exactly when it is not. + +Reading the stored SCSS against the target sources answers before the bump, +in a second, and names the attachment. Nothing is written here. + +What it can miss +---------------- +Which files a bundle really contains is decided by Odoo at runtime. This +compares against every SCSS of the INSTALLED modules of the target version, +which is wider: a variable defined in a module file that the bundle does not +include would be counted as defined. That direction is deliberate — it under- +reports rather than crying wolf on every customization. + +Exit codes: 0 nothing to report, 1 customizations at risk, 2 tool failure. +""" + +import argparse +import glob +import os +import re +import subprocess +import sys + +sys.path.append( + os.path.normpath(os.path.join(os.path.dirname(__file__), "..", "..", "..")) +) + +try: + from script.todo.todo_i18n import t +except Exception: # pragma: no cover - repli si i18n indisponible + + def t(key: str) -> str: + return key + + +# Une variable SCSS commence par $ ; celles dont le nom commence par un tiret +# sont locales au fichier par convention Odoo ($-seen, $-font-numbers) et se +# définissent toujours au-dessus de leur usage. +RE_ANY = re.compile(r"\$([a-zA-Z][\w-]*)") + + +def used_names(content): + """Noms LUS par le fichier, sans les liaisons. + + Un « $nom » suivi d'un deux-points est toujours une liaison — déclaration, + valeur par défaut, ou argument nommé d'un @include — jamais la lecture + d'une variable. Sans cette réserve, « @include o-position-absolute( + $right: 50%) » se lisait comme l'usage d'un $right inexistant. + + Le test se fait ici et non par une négation dans le motif : une négation + en tête de motif fait rétrograder le nom pour satisfaire la condition, et + rapporte « $botto » au lieu d'écarter « $bottom ». Mesuré. + """ + names = set() + for match in RE_ANY.finditer(content): + if content[match.end() :].lstrip(" \t").startswith(":"): + continue + names.add(match.group(1)) + return names + + +RE_DEF = re.compile(r"^\s*\$([a-zA-Z][\w-]*)\s*:", re.M) + +# Une variable peut aussi être LIÉE sans être définie : paramètre de mixin ou +# de fonction, variable de boucle. Les ignorer donnait six faux positifs sur +# le seul fichier mesuré — $bottom, $counter, $off, $on, $right, $value — et +# un détecteur qui crie à tort finit ignoré, ce qui vaut moins que rien. +RE_PARAM = re.compile(r"@(?:mixin|function)\s+[\w-]+\s*\(([^)]*)\)") +RE_EACH = re.compile(r"@each\s+([^i]+?)\s+in\s", re.S) +RE_FOR = re.compile(r"@for\s+\$([\w-]+)\s+from\s") + + +def bound_names(content): + """Noms liés par le fichier lui-même, hors définitions « $x: … ».""" + names = set() + for group in RE_PARAM.findall(content): + names.update(RE_ANY.findall(group)) + for group in RE_EACH.findall(content): + names.update(RE_ANY.findall(group)) + names.update(RE_FOR.findall(content)) + return names + + +def run_psql(database, sql): + """Interroger la base, lecture seule garantie par le serveur.""" + env = os.environ.copy() + env["PGOPTIONS"] = ( + "-c default_transaction_read_only=on -c statement_timeout=60000" + ) + env["PSQLRC"] = "" + done = subprocess.run( + [ + "psql", + "-X", + "-w", + "-v", + "ON_ERROR_STOP=1", + "-d", + database, + "-tAF", + "\x1f", + "-c", + sql, + ], + capture_output=True, + text=True, + env=env, + ) + if done.returncode: + raise RuntimeError(done.stderr.strip() or "psql failed") + return [line.split("\x1f") for line in done.stdout.splitlines() if line] + + +def filestore_dir(database, override=None): + if override: + return override + return os.path.join( + os.path.expanduser("~"), + ".local", + "share", + "Odoo", + "filestore", + database, + ) + + +def custom_scss(database, filestore): + """[(id, url, contenu)] pour chaque SCSS personnalisé lisible.""" + rows = run_psql( + database, + "SELECT id, url, COALESCE(store_fname, '')," + " COALESCE(encode(db_datas, 'escape'), '') FROM ir_attachment" + " WHERE url LIKE '%.custom.%' AND url NOT LIKE '%.css'" + " ORDER BY id;", + ) + lst = [] + for row in rows: + if len(row) < 4: + continue + att_id, url, store_fname, db_datas = row[0], row[1], row[2], row[3] + content = "" + if store_fname: + path = os.path.join(filestore, store_fname) + if os.path.isfile(path): + with open(path, "r", encoding="utf-8", errors="replace") as h: + content = h.read() + elif db_datas: + content = db_datas + if content: + lst.append((int(att_id), url, content)) + return lst + + +def installed_modules(database): + return { + row[0] + for row in run_psql( + database, + "SELECT name FROM ir_module_module WHERE state = 'installed';", + ) + if row and row[0] + } + + +def defined_in_sources(version_dir, lst_module): + """Variables définies par les SCSS des modules installés de la cible. + + Le nom du module est le répertoire qui contient « static » : c'est ce qui + permet de ne pas compter les variables d'un module absent de la base, dont + aucune ligne n'atteindra jamais un bundle. + """ + defined = set() + pattern = os.path.join(version_dir, "**", "static", "**", "*.scss") + for path in glob.iglob(pattern, recursive=True): + parts = path.split(os.sep) + try: + module = parts[parts.index("static") - 1] + except (ValueError, IndexError): + continue + if module not in lst_module: + continue + try: + with open(path, "r", encoding="utf-8", errors="replace") as handle: + defined.update(RE_DEF.findall(handle.read())) + except OSError: + continue + return defined + + +def analyse(database, version_dir, filestore=None): + """[(id, url, [variables absentes])] pour les personnalisations à risque.""" + filestore = filestore_dir(database, filestore) + lst_module = installed_modules(database) + defined = defined_in_sources(version_dir, lst_module) + lst_finding = [] + for att_id, url, content in custom_scss(database, filestore): + own = set(RE_DEF.findall(content)) | bound_names(content) + missing = sorted( + { + name + for name in used_names(content) + if name not in own and name not in defined + } + ) + if missing: + lst_finding.append((att_id, url, missing)) + return lst_finding + + +def render(lst_finding, database, version_dir): + if not lst_finding: + return ( + f"✅ -> {t('No customized SCSS is at risk in')} {version_dir}.\n" + ) + lines = [ + f"⚠️ {len(lst_finding)}" + f" {t('customized SCSS use(s) a variable that')} {version_dir}" + f" {t('no longer defines: the bundle will not compile.')}" + ] + for att_id, url, missing in lst_finding: + lines.append(f" - id={att_id} {url}") + lines.append(f" {', '.join('$' + m for m in missing)}") + lines += [ + f" {t('Each one is a copy frozen on an older version. Dropping it')}" + f" {t('restores the module file:')}", + f" ./odoo_bin.sh shell -c ./config.conf -d {database} <<'PY'", + ] + for _att_id, url, _missing in lst_finding: + base, bundle = split_custom_url(url) + lines.append( + f" env['web_editor.assets'].reset_asset(" + f"'{base}', '{bundle}')" + ) + lines += [ + " env.cr.commit()", + " PY", + f" {t('Read it first: what it holds beyond the stale variable is')}" + f" {t('a real customization, to re-apply as a small file.')}", + ] + return "\n".join(lines) + "\n" + + +def split_custom_url(url): + """« /a/b.custom.web.assets_frontend.scss » -> (« /a/b.scss », bundle). + + C'est la convention d'Odoo : le nom du bundle est encastré dans celui du + fichier. La reconstruire évite de faire deviner à l'utilisateur les deux + arguments de reset_asset au moment où il veut juste réparer. + """ + directory, name = os.path.split(url) + head, _sep, tail = name.partition(".custom.") + bundle = tail[:-5] if tail.endswith(".scss") else tail + return os.path.join(directory, head + ".scss"), bundle + + +def main(argv=None): + parser = argparse.ArgumentParser( + description=( + "Predict which customized SCSS will break on the target version" + " (read-only)." + ) + ) + parser.add_argument("-d", "--database", required=True) + parser.add_argument( + "-t", + "--target_version", + required=True, + help="target Odoo source directory, e.g. odoo13.0", + ) + parser.add_argument( + "--filestore", + default=None, + help="filestore directory (default: ~/.local/share/Odoo/filestore/)", + ) + config = parser.parse_args(argv) + + if not os.path.isdir(config.target_version): + print( + f"❌ {t('Target version directory not found')} :" + f" '{config.target_version}'" + ) + return 2 + try: + lst_finding = analyse( + config.database, config.target_version, config.filestore + ) + except RuntimeError as exc: + print(f"❌ {exc}") + return 2 + print(render(lst_finding, config.database, config.target_version)) + return 1 if lst_finding else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/script/todo/todo_i18n.py b/script/todo/todo_i18n.py index 74d9734..aa611d1 100644 --- a/script/todo/todo_i18n.py +++ b/script/todo/todo_i18n.py @@ -4790,6 +4790,38 @@ TRANSLATIONS = { "fr": "Conservés. Rien n'a été désinstallé.", "en": "Kept. Nothing was uninstalled.", }, + "No customized SCSS is at risk in": { + "fr": "Aucun SCSS personnalisé n'est en danger dans", + "en": "No customized SCSS is at risk in", + }, + "customized SCSS use(s) a variable that": { + "fr": "SCSS personnalisé(s) utilisent une variable que", + "en": "customized SCSS use(s) a variable that", + }, + "no longer defines: the bundle will not compile.": { + "fr": "ne définit plus : le bundle ne compilera pas.", + "en": "no longer defines: the bundle will not compile.", + }, + "Each one is a copy frozen on an older version. Dropping it": { + "fr": "Chacun est une copie figée sur une version antérieure. L'abandonner", + "en": "Each one is a copy frozen on an older version. Dropping it", + }, + "restores the module file:": { + "fr": "rend le fichier du module :", + "en": "restores the module file:", + }, + "Read it first: what it holds beyond the stale variable is": { + "fr": "Le lire d'abord : ce qu'il porte au-delà de la variable périmée est", + "en": "Read it first: what it holds beyond the stale variable is", + }, + "a real customization, to re-apply as a small file.": { + "fr": "une vraie personnalisation, à réappliquer en petit fichier.", + "en": "a real customization, to re-apply as a small file.", + }, + "Read the customizations above before continuing.": { + "fr": "Lire les personnalisations ci-dessus avant de continuer.", + "en": "Read the customizations above before continuing.", + }, "Nothing to decide yet": { "fr": "Rien à décider pour l'instant", "en": "Nothing to decide yet", diff --git a/script/todo/todo_upgrade.py b/script/todo/todo_upgrade.py index 0072e92..8ea6765 100755 --- a/script/todo/todo_upgrade.py +++ b/script/todo/todo_upgrade.py @@ -1641,6 +1641,22 @@ class TodoUpgrade: if status == 1: self.prompt_cow_prediction(database_name, start_version + 1) + # Même prédiction, autre matière : un SCSS personnalisé est lui aussi + # une copie figée, et un palier renomme aussi des variables. Celle-ci + # ne se voit qu'à l'écran d'une page — « Style error », sans dire quel + # fichier ni depuis quand — et seulement une fois le palier passé. + status_scss, _cmd, _out = self.todo_upgrade_execute( + f"{PYTHON_BIN} ./script/odoo/migration/check_stale_scss.py" + f" -d {database_name} -t odoo{start_version + 1}.0", + get_output=True, + wait_at_error=False, + ) + if status_scss == 1: + self.ask_gate( + f"💬 {t('Read the customizations above before continuing.')}" + f" {t('(b = go back to a previous step)')} : " + ) + msg = "3 - Clean up database before data migration" self.print_step(msg) self.add_comment_progression(msg) diff --git a/test/test_check_stale_scss.py b/test/test_check_stale_scss.py new file mode 100755 index 0000000..6a6f0f0 --- /dev/null +++ b/test/test_check_stale_scss.py @@ -0,0 +1,160 @@ +#!/usr/bin/env python3 +# © 2021-2026 TechnoLibre (http://www.technolibre.ca) +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl) + +"""Un SCSS personnalisé est une copie figée, comme une vue COW. + +Personnaliser un site écrit du SCSS dans `ir_attachment`. Cette copie est +figée le jour où elle est écrite et continue d'employer les variables de +CETTE version. Un module peut les renommer d'un palier à l'autre : mesuré, +`website/.../primary_variables.scss` déclarait `$o-theme-font-number` en 12.0 +et a remplacé tout le mécanisme en 13.0. Une personnalisation de 2020 +demandait encore l'ancien nom, et le bundle s'arrêtait dessus. + +Ce qui rend un tel détecteur utilisable n'est pas de trouver — c'est de ne +pas crier à tort. Sur le seul fichier mesuré, la première version rapportait +dix noms dont six n'étaient pas des manques : paramètres de mixin, variables +de boucle, arguments nommés d'`@include`. Ces tests portent surtout là-dessus. +""" + +import os +import sys +import unittest + +REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.insert(0, os.path.join(REPO, "script", "odoo", "migration")) + +import check_stale_scss as scss # noqa: E402 + + +class TestWhatCountsAsAUse(unittest.TestCase): + def test_a_plain_dereference_is_a_use(self): + self.assertIn("color", scss.used_names("a { b: $color; }")) + + def test_a_declaration_is_not_a_use(self): + self.assertNotIn("color", scss.used_names("$color: red;")) + + def test_a_named_include_argument_is_not_a_use(self): + # LE faux positif mesuré : « @include o-position-absolute( + # $right: 50%) » se lisait comme l'usage d'un $right inexistant. + source = "@include o-position-absolute($right: 50%, $left: 50%);" + self.assertEqual(scss.used_names(source), set()) + + def test_a_default_value_is_not_a_use(self): + self.assertNotIn("size", scss.used_names("$size: 3 !default;")) + + def test_the_name_is_not_truncated_to_dodge_the_colon(self): + # Une négation en tête de motif fait rétrograder le nom : le premier + # essai rapportait « $botto » au lieu d'écarter « $bottom ». + names = scss.used_names("a { b: $bottom-margin; }") + self.assertEqual(names, {"bottom-margin"}) + + def test_a_use_after_a_declaration_still_counts(self): + self.assertEqual(scss.used_names("$a: $b;"), {"b"}) + + +class TestWhatTheFileBindsItself(unittest.TestCase): + def test_mixin_parameters_are_bound(self): + source = "@mixin thing($on, $off: 2) { a: $on; b: $off; }" + self.assertEqual(scss.bound_names(source), {"on", "off"}) + + def test_function_parameters_are_bound(self): + source = "@function f($value) { @return $value; }" + self.assertIn("value", scss.bound_names(source)) + + def test_each_loop_variables_are_bound(self): + source = "@each $key, $val in $map { a: $key; }" + bound = scss.bound_names(source) + self.assertIn("key", bound) + self.assertIn("val", bound) + + def test_for_loop_variables_are_bound(self): + source = "@for $counter from 1 through 3 { a: $counter; }" + self.assertIn("counter", scss.bound_names(source)) + + +class TestTheUrlIsSplitForTheFix(unittest.TestCase): + """reset_asset veut le fichier ET le bundle ; l'URL porte les deux.""" + + def test_a_frontend_customization(self): + base, bundle = scss.split_custom_url( + "/website/static/src/scss/website.custom.web.assets_frontend.scss" + ) + self.assertEqual(base, "/website/static/src/scss/website.scss") + self.assertEqual(bundle, "web.assets_frontend") + + def test_a_common_customization(self): + base, bundle = scss.split_custom_url( + "/website/static/src/scss/options/colors/" + "user_theme_color_palette.custom.web.assets_common.scss" + ) + self.assertTrue(base.endswith("user_theme_color_palette.scss")) + self.assertEqual(bundle, "web.assets_common") + + +class TestTheReport(unittest.TestCase): + def setUp(self): + from script.todo import todo_i18n + + self.addCleanup( + setattr, todo_i18n, "_current_lang", todo_i18n._current_lang + ) + todo_i18n._current_lang = "en" + + def test_nothing_at_risk_says_so(self): + text = scss.render([], "db", "odoo13.0") + self.assertIn("✅", text) + + def test_a_finding_names_the_attachment_and_the_variable(self): + text = scss.render( + [ + ( + 1110, + "/a/b.custom.web.assets_frontend.scss", + ["o-theme-font-number"], + ) + ], + "db", + "odoo13.0", + ) + self.assertIn("1110", text) + self.assertIn("$o-theme-font-number", text) + + def test_it_prints_a_command_that_can_be_pasted(self): + # Un diagnostic sans le geste qui répare oblige à rechercher les deux + # arguments de reset_asset au pire moment. + text = scss.render( + [(1110, "/a/b.custom.web.assets_frontend.scss", ["x"])], + "mydb", + "odoo13.0", + ) + self.assertIn("reset_asset('/a/b.scss', 'web.assets_frontend')", text) + self.assertIn("-d mydb", text) + + def test_it_warns_before_dropping(self): + text = scss.render( + [(1110, "/a/b.custom.web.assets_frontend.scss", ["x"])], + "db", + "odoo13.0", + ) + self.assertIn("real customization", text) + + +class TestTheMigrationRunsIt(unittest.TestCase): + def test_it_is_checked_at_the_same_place_as_the_cow_views(self): + # Les deux prédictions valent avant le palier, pas après : c'est le + # seul moment où l'on peut encore arbitrer sans avoir tout rejoué. + import inspect + + from script.todo.todo_upgrade import TodoUpgrade + + source = inspect.getsource(TodoUpgrade.execute_odoo_upgrade) + self.assertIn("check_stale_scss.py", source) + self.assertLess( + source.index("check_stale_scss.py"), + source.index("4 - Upgrade version with OpenUpgrade"), + ) + + +if __name__ == "__main__": + unittest.main()