diff --git a/script/shell_script_odoo/brin_advisor.py b/script/shell_script_odoo/brin_advisor.py new file mode 100644 index 0000000..2f27518 --- /dev/null +++ b/script/shell_script_odoo/brin_advisor.py @@ -0,0 +1,191 @@ +#!/usr/bin/env python3 +""" +brin_advisor.py — Recommends the optimal index type for an Odoo 18 model + +Usage (via odoo-bin shell): + source .venv/bin/activate + echo "MODEL='account.move.line'" | cat - brin_advisor.py \ + | odoo-bin shell -d my_database --no-http + + # Or with export: + export MODEL=mail.message + cat brin_advisor.py | odoo-bin shell -d my_database --no-http +""" + +import os + +# ── Constants ───────────────────────────────────────────────────────────────── + +DATE_COLUMNS = [ + "date", + "create_date", + "write_date", + "invoice_date", + "scheduled_date", + "date_done", + "in_date", + "expiration_date", +] + +THRESHOLDS = [ + (0.9, "✅ BRIN ideal", "\033[0;32m"), + (0.5, "⚠️ BRIN acceptable", "\033[1;33m"), + (0.1, "❌ Prefer B-tree", "\033[0;31m"), + (-1.0, "❌ B-tree mandatory", "\033[0;31m"), +] + +RESET = "\033[0m" +BOLD = "\033[1m" +CYAN = "\033[0;36m" +GREEN = "\033[0;32m" +YELLOW = "\033[1;33m" +RED = "\033[0;31m" + + +# ── Helpers ─────────────────────────────────────────────────────────────────── + + +def recommend(correlation): + for threshold, label, color in THRESHOLDS: + if correlation >= threshold: + return label, color + return THRESHOLDS[-1][1], THRESHOLDS[-1][2] + + +def model_to_table(model_name): + """ + Resolve model name to table via the Odoo ORM — more reliable than a plain + replace('.', '_') since some models override _table explicitly. + """ + if model_name not in env: + # Fallback: standard Odoo naming convention + return model_name.replace(".", "_") + return env[model_name]._table + + +# ── Model resolution ────────────────────────────────────────────────────────── +# Priority: MODEL variable defined before the pipe, then environment variable +model_name = ( + globals().get("MODEL") or os.environ.get("MODEL") or "account.move.line" +) +table = model_to_table(model_name) +cr = env.cr # PostgreSQL cursor provided by odoo-bin shell + +print(f"\n{BOLD}{CYAN}=== BRIN Advisor for: {model_name} ==={RESET}\n") +print(f" Model : {model_name}") +print(f" Table : {table}\n") + +# ── Check table exists ──────────────────────────────────────────────────────── +cr.execute( + """ + SELECT COUNT(*) FROM information_schema.tables + WHERE table_schema = 'public' AND table_name = %s +""", + (table,), +) +if cr.fetchone()[0] == 0: + print(f"{RED}❌ Table '{table}' not found in the database.{RESET}\n") + raise SystemExit(1) + +# ── Automatic ANALYZE ───────────────────────────────────────────────────────── +# odoo-bin shell runs inside an open transaction — ANALYZE is not allowed +# inside a transaction, so we temporarily switch to AUTOCOMMIT. +print( + f" {YELLOW}⟳ Running ANALYZE on {table}...{RESET}", end=" ", flush=True +) +with env.cr._cnx.cursor() as auto_cr: + old_isolation = env.cr._cnx.isolation_level + env.cr._cnx.set_isolation_level(0) # AUTOCOMMIT + auto_cr.execute( + f"ANALYZE {table}" + ) # table name validated via ORM, no injection risk + env.cr._cnx.set_isolation_level(old_isolation) +print(f"{GREEN}OK{RESET}\n") + +# ── Table size ──────────────────────────────────────────────────────────────── +cr.execute( + """ + SELECT + pg_size_pretty(pg_total_relation_size(%s::regclass)), + reltuples::bigint + FROM pg_class WHERE relname = %s +""", + (table, table), +) +row = cr.fetchone() +if row: + size_pretty, row_count = row + print( + f" Size : {BOLD}{size_pretty}{RESET} (~{row_count:,} estimated rows)\n" + ) + +# ── Correlation analysis ────────────────────────────────────────────────────── +cr.execute( + """ + SELECT attname, correlation + FROM pg_stats + WHERE tablename = %s + AND attname = ANY(%s) + ORDER BY ABS(correlation) DESC NULLS LAST +""", + (table, DATE_COLUMNS), +) +stats = cr.fetchall() + +print(f"{BOLD}Date columns found:{RESET}") +print("─" * 70) +print(f" {'COLUMN':<25} {'CORRELATION':<14} {'IDX SIZE':<12} RECOMMENDATION") +print("─" * 70) + +if not stats: + print(f" {YELLOW}No standard date columns found on this table.{RESET}") +else: + for col, corr in stats: + if corr is None: + continue + # Size of existing BRIN index on this column, if any + cr.execute( + """ + SELECT pg_size_pretty(pg_relation_size(idx.indexrelid)) + FROM pg_index idx + JOIN pg_class i ON i.oid = idx.indexrelid + JOIN pg_attribute a + ON a.attrelid = %s::regclass + AND a.attnum = ANY(idx.indkey) + AND a.attname = %s + JOIN pg_am am ON am.oid = i.relam AND am.amname = 'brin' + WHERE idx.indrelid = %s::regclass + LIMIT 1 + """, + (table, col, table), + ) + idx_row = cr.fetchone() + idx_size = idx_row[0] if idx_row else "none" + + label, color = recommend(corr) + print( + f" {col:<25} {corr:<14.4f} {idx_size:<12} {color}{label}{RESET}" + ) + +print("─" * 70) + +# ── Existing BRIN indexes ───────────────────────────────────────────────────── +print(f"\n{BOLD}Existing BRIN indexes:{RESET}") +cr.execute( + """ + SELECT indexname, indexdef + FROM pg_indexes + WHERE tablename = %s + AND indexdef LIKE '%%USING brin%%' +""", + (table,), +) +brin_indexes = cr.fetchall() + +if not brin_indexes: + print(f" {YELLOW}No BRIN indexes found on this table.{RESET}") +else: + for iname, _ in brin_indexes: + print(f" {GREEN}✔{RESET} {iname}") + +print() diff --git a/script/shell_script_odoo/brin_cluster.py b/script/shell_script_odoo/brin_cluster.py new file mode 100644 index 0000000..1bcbc57 --- /dev/null +++ b/script/shell_script_odoo/brin_cluster.py @@ -0,0 +1,286 @@ +#!/usr/bin/env python3 +""" +brin_cluster.py — Reorders table rows physically to optimize BRIN indexes. + +Strategy: + 1. Check correlation of candidate columns via pg_stats + 2. Pick the best column (highest absolute correlation, ideally create_date) + 3. Create a temporary B-tree index on that column + 4. CLUSTER the table on that index (rewrites rows in physical order) + 5. ANALYZE to refresh statistics + 6. Drop the temporary B-tree index + 7. Report the new correlation + +WARNING: + - CLUSTER acquires an ACCESS EXCLUSIVE lock — the table is fully locked. + - On large tables, this can take minutes to hours. + - Run during a maintenance window or off-peak hours. + - Always pg_dump before running on production. + +Usage: + export MODEL=account.move.line + cat brin_cluster.py | odoo-bin shell -d my_database --no-http + + # Dry run (no changes, just shows what would be done): + export MODEL=account.move.line DRY_RUN=1 + cat brin_cluster.py | odoo-bin shell -d my_database --no-http +""" + +import os +import time + +# ── Constants ───────────────────────────────────────────────────────────────── + +# Columns considered as clustering candidates, in priority order. +# create_date is first: it's never updated, so correlation is always perfect. +CANDIDATE_COLUMNS = [ + "create_date", + "date", + "invoice_date", + "write_date", + "scheduled_date", + "date_done", + "in_date", +] + +BRIN_IDEAL_THRESHOLD = 0.9 + +RESET = "\033[0m" +BOLD = "\033[1m" +CYAN = "\033[0;36m" +GREEN = "\033[0;32m" +YELLOW = "\033[1;33m" +RED = "\033[0;31m" + + +# ── Config ──────────────────────────────────────────────────────────────────── + +model_name = ( + globals().get("MODEL") or os.environ.get("MODEL") or "account.move.line" +) +dry_run = bool(globals().get("DRY_RUN") or os.environ.get("DRY_RUN")) + +if model_name not in env: + table = model_name.replace(".", "_") +else: + table = env[model_name]._table + +cr = env.cr +cnx = env.cr._cnx + +print(f"\n{BOLD}{CYAN}=== BRIN Cluster for: {model_name} ==={RESET}") +print(f" Table : {table}") +print(f" Dry run : {dry_run}\n") + + +# ── Helpers ─────────────────────────────────────────────────────────────────── + + +def autocommit(sql, params=None): + """Execute a DDL statement outside the current transaction (AUTOCOMMIT).""" + old = cnx.isolation_level + cnx.set_isolation_level(0) + with cnx.cursor() as c: + c.execute(sql, params) + cnx.set_isolation_level(old) + + +def get_correlations(): + cr.execute( + """ + SELECT attname, correlation + FROM pg_stats + WHERE tablename = %s + AND attname = ANY(%s) + AND correlation IS NOT NULL + ORDER BY ABS(correlation) DESC + """, + (table, CANDIDATE_COLUMNS), + ) + return {row[0]: row[1] for row in cr.fetchall()} + + +def table_size(): + cr.execute( + """ + SELECT + pg_size_pretty(pg_total_relation_size(%s::regclass)), + reltuples::bigint + FROM pg_class WHERE relname = %s + """, + (table, table), + ) + return cr.fetchone() + + +def index_exists(index_name): + cr.execute( + """ + SELECT COUNT(*) FROM pg_indexes + WHERE tablename = %s AND indexname = %s + """, + (table, index_name), + ) + return cr.fetchone()[0] > 0 + + +# ── Step 1: ANALYZE to get fresh stats ─────────────────────────────────────── + +print(f" {YELLOW}⟳ ANALYZE {table}...{RESET}", end=" ", flush=True) +autocommit(f"ANALYZE {table}") +print(f"{GREEN}OK{RESET}\n") + +size_pretty, row_count = table_size() +print( + f" Size : {BOLD}{size_pretty}{RESET} (~{row_count:,} estimated rows)\n" +) + +# ── Step 2: Check current correlations ─────────────────────────────────────── + +correlations = get_correlations() + +if not correlations: + print(f"{RED}❌ No date columns found on {table}. Aborting.{RESET}\n") + raise SystemExit(1) + +print(f"{BOLD}Current correlations:{RESET}") +for col, corr in sorted(correlations.items(), key=lambda x: -abs(x[1])): + if abs(corr) >= BRIN_IDEAL_THRESHOLD: + icon = f"{GREEN}✅ BRIN ideal{RESET}" + else: + icon = f"{YELLOW}⚠️ needs clustering{RESET}" + print(f" {col:<25} {corr:+.4f} {icon}") + +# ── Step 3: Pick the best clustering column ─────────────────────────────────── + +# Only cluster if at least one *candidate* column is suboptimal. +# Note: a column like `date` (user-editable) may never reach ideal correlation +# regardless of clustering — that's expected and not a reason to cluster. +suboptimal = { + col: corr + for col, corr in correlations.items() + if abs(corr) < BRIN_IDEAL_THRESHOLD +} +ideal = { + col: corr + for col, corr in correlations.items() + if abs(corr) >= BRIN_IDEAL_THRESHOLD +} + +if suboptimal: + print(f"\n {YELLOW}⚠️ Suboptimal columns: {', '.join(suboptimal)}{RESET}") + print( + f" {CYAN}Note: columns like 'date' or 'invoice_date' may stay suboptimal" + ) + print( + f" after clustering if users enter backdated records — that's expected.{RESET}" + ) +else: + print( + f"\n{GREEN}✅ All columns already have ideal correlation. No clustering needed.{RESET}\n" + ) + raise SystemExit(0) + +# Pick the best clustering column: highest absolute correlation among candidates. +# Prefer create_date first (never updated → guaranteed monotonic order), +# then fall back to the column with highest abs(correlation). +cluster_col = None +for preferred in CANDIDATE_COLUMNS: + if preferred in ideal: # already ideal → good anchor for physical order + cluster_col = preferred + break +if not cluster_col: + # All candidates are suboptimal — pick the least bad one + cluster_col = max(suboptimal, key=lambda c: abs(suboptimal[c])) + +if not cluster_col: + print(f"{RED}❌ No suitable clustering column found. Aborting.{RESET}\n") + raise SystemExit(1) + +print( + f"\n {BOLD}Clustering column chosen:{RESET} {cluster_col} " + f"(current correlation: {correlations[cluster_col]:+.4f})\n" +) + +# ── Step 4: Create temporary B-tree index ──────────────────────────────────── + +tmp_index = f"_brin_cluster_tmp_{table}_{cluster_col}" + +if index_exists(tmp_index): + print( + f" {YELLOW}⚠️ Temporary index already exists, dropping it first...{RESET}" + ) + if not dry_run: + autocommit(f'DROP INDEX IF EXISTS "{tmp_index}"') + +print( + f" {YELLOW}⟳ Creating temporary B-tree index on {cluster_col}...{RESET}", + end=" ", + flush=True, +) +if not dry_run: + autocommit(f'CREATE INDEX "{tmp_index}" ON "{table}" ("{cluster_col}")') + print(f"{GREEN}OK{RESET}") +else: + print(f"{CYAN}[DRY RUN]{RESET}") + +# ── Step 5: CLUSTER ─────────────────────────────────────────────────────────── + +print(f"\n {YELLOW}⟳ CLUSTER {table} on {cluster_col}...{RESET}") +print( + f" {YELLOW} (ACCESS EXCLUSIVE lock — table unavailable during this operation){RESET}", + end=" ", + flush=True, +) + +if not dry_run: + t0 = time.time() + autocommit(f'CLUSTER "{table}" USING "{tmp_index}"') + elapsed = time.time() - t0 + print(f"{GREEN}OK{RESET} ({elapsed:.1f}s)") +else: + print(f"{CYAN}[DRY RUN]{RESET}") + +# ── Step 6: Drop temporary index ───────────────────────────────────────────── + +print( + f"\n {YELLOW}⟳ Dropping temporary index...{RESET}", end=" ", flush=True +) +if not dry_run: + autocommit(f'DROP INDEX IF EXISTS "{tmp_index}"') + print(f"{GREEN}OK{RESET}") +else: + print(f"{CYAN}[DRY RUN]{RESET}") + +# ── Step 7: ANALYZE again + report new correlations ────────────────────────── + +print( + f"\n {YELLOW}⟳ ANALYZE after clustering...{RESET}", end=" ", flush=True +) +if not dry_run: + autocommit(f"ANALYZE {table}") + print(f"{GREEN}OK{RESET}\n") + + new_correlations = get_correlations() + + print(f"{BOLD}Correlations after clustering:{RESET}") + print("─" * 55) + print(f" {'COLUMN':<25} {'BEFORE':>8} {'AFTER':>8} RESULT") + print("─" * 55) + for col in sorted(correlations.keys()): + before = correlations.get(col, 0.0) + after = new_correlations.get(col, 0.0) + delta = after - before + if abs(after) >= BRIN_IDEAL_THRESHOLD: + result = f"{GREEN}✅ BRIN ideal{RESET}" + else: + result = f"{YELLOW}⚠️ still suboptimal{RESET}" + arrow = "↑" if delta > 0.01 else ("↓" if delta < -0.01 else "→") + print( + f" {col:<25} {before:>+8.4f} {after:>+8.4f} {arrow} {result}" + ) + print("─" * 55) +else: + print(f"{CYAN}[DRY RUN]{RESET}") + +print(f"\n{GREEN}✅ Done.{RESET}\n")