[ADD] brin_advisor brin_cluster
This commit is contained in:
parent
e9f14c2af9
commit
624c2d89e9
2 changed files with 477 additions and 0 deletions
191
script/shell_script_odoo/brin_advisor.py
Normal file
191
script/shell_script_odoo/brin_advisor.py
Normal file
|
|
@ -0,0 +1,191 @@
|
|||
#!/usr/bin/env python3
|
||||
"""
|
||||
brin_advisor.py — Recommends the optimal index type for an Odoo 18 model
|
||||
|
||||
Usage (via odoo-bin shell):
|
||||
source .venv/bin/activate
|
||||
echo "MODEL='account.move.line'" | cat - brin_advisor.py \
|
||||
| odoo-bin shell -d my_database --no-http
|
||||
|
||||
# Or with export:
|
||||
export MODEL=mail.message
|
||||
cat brin_advisor.py | odoo-bin shell -d my_database --no-http
|
||||
"""
|
||||
|
||||
import os
|
||||
|
||||
# ── Constants ─────────────────────────────────────────────────────────────────
|
||||
|
||||
DATE_COLUMNS = [
|
||||
"date",
|
||||
"create_date",
|
||||
"write_date",
|
||||
"invoice_date",
|
||||
"scheduled_date",
|
||||
"date_done",
|
||||
"in_date",
|
||||
"expiration_date",
|
||||
]
|
||||
|
||||
THRESHOLDS = [
|
||||
(0.9, "✅ BRIN ideal", "\033[0;32m"),
|
||||
(0.5, "⚠️ BRIN acceptable", "\033[1;33m"),
|
||||
(0.1, "❌ Prefer B-tree", "\033[0;31m"),
|
||||
(-1.0, "❌ B-tree mandatory", "\033[0;31m"),
|
||||
]
|
||||
|
||||
RESET = "\033[0m"
|
||||
BOLD = "\033[1m"
|
||||
CYAN = "\033[0;36m"
|
||||
GREEN = "\033[0;32m"
|
||||
YELLOW = "\033[1;33m"
|
||||
RED = "\033[0;31m"
|
||||
|
||||
|
||||
# ── Helpers ───────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def recommend(correlation):
|
||||
for threshold, label, color in THRESHOLDS:
|
||||
if correlation >= threshold:
|
||||
return label, color
|
||||
return THRESHOLDS[-1][1], THRESHOLDS[-1][2]
|
||||
|
||||
|
||||
def model_to_table(model_name):
|
||||
"""
|
||||
Resolve model name to table via the Odoo ORM — more reliable than a plain
|
||||
replace('.', '_') since some models override _table explicitly.
|
||||
"""
|
||||
if model_name not in env:
|
||||
# Fallback: standard Odoo naming convention
|
||||
return model_name.replace(".", "_")
|
||||
return env[model_name]._table
|
||||
|
||||
|
||||
# ── Model resolution ──────────────────────────────────────────────────────────
|
||||
# Priority: MODEL variable defined before the pipe, then environment variable
|
||||
model_name = (
|
||||
globals().get("MODEL") or os.environ.get("MODEL") or "account.move.line"
|
||||
)
|
||||
table = model_to_table(model_name)
|
||||
cr = env.cr # PostgreSQL cursor provided by odoo-bin shell
|
||||
|
||||
print(f"\n{BOLD}{CYAN}=== BRIN Advisor for: {model_name} ==={RESET}\n")
|
||||
print(f" Model : {model_name}")
|
||||
print(f" Table : {table}\n")
|
||||
|
||||
# ── Check table exists ────────────────────────────────────────────────────────
|
||||
cr.execute(
|
||||
"""
|
||||
SELECT COUNT(*) FROM information_schema.tables
|
||||
WHERE table_schema = 'public' AND table_name = %s
|
||||
""",
|
||||
(table,),
|
||||
)
|
||||
if cr.fetchone()[0] == 0:
|
||||
print(f"{RED}❌ Table '{table}' not found in the database.{RESET}\n")
|
||||
raise SystemExit(1)
|
||||
|
||||
# ── Automatic ANALYZE ─────────────────────────────────────────────────────────
|
||||
# odoo-bin shell runs inside an open transaction — ANALYZE is not allowed
|
||||
# inside a transaction, so we temporarily switch to AUTOCOMMIT.
|
||||
print(
|
||||
f" {YELLOW}⟳ Running ANALYZE on {table}...{RESET}", end=" ", flush=True
|
||||
)
|
||||
with env.cr._cnx.cursor() as auto_cr:
|
||||
old_isolation = env.cr._cnx.isolation_level
|
||||
env.cr._cnx.set_isolation_level(0) # AUTOCOMMIT
|
||||
auto_cr.execute(
|
||||
f"ANALYZE {table}"
|
||||
) # table name validated via ORM, no injection risk
|
||||
env.cr._cnx.set_isolation_level(old_isolation)
|
||||
print(f"{GREEN}OK{RESET}\n")
|
||||
|
||||
# ── Table size ────────────────────────────────────────────────────────────────
|
||||
cr.execute(
|
||||
"""
|
||||
SELECT
|
||||
pg_size_pretty(pg_total_relation_size(%s::regclass)),
|
||||
reltuples::bigint
|
||||
FROM pg_class WHERE relname = %s
|
||||
""",
|
||||
(table, table),
|
||||
)
|
||||
row = cr.fetchone()
|
||||
if row:
|
||||
size_pretty, row_count = row
|
||||
print(
|
||||
f" Size : {BOLD}{size_pretty}{RESET} (~{row_count:,} estimated rows)\n"
|
||||
)
|
||||
|
||||
# ── Correlation analysis ──────────────────────────────────────────────────────
|
||||
cr.execute(
|
||||
"""
|
||||
SELECT attname, correlation
|
||||
FROM pg_stats
|
||||
WHERE tablename = %s
|
||||
AND attname = ANY(%s)
|
||||
ORDER BY ABS(correlation) DESC NULLS LAST
|
||||
""",
|
||||
(table, DATE_COLUMNS),
|
||||
)
|
||||
stats = cr.fetchall()
|
||||
|
||||
print(f"{BOLD}Date columns found:{RESET}")
|
||||
print("─" * 70)
|
||||
print(f" {'COLUMN':<25} {'CORRELATION':<14} {'IDX SIZE':<12} RECOMMENDATION")
|
||||
print("─" * 70)
|
||||
|
||||
if not stats:
|
||||
print(f" {YELLOW}No standard date columns found on this table.{RESET}")
|
||||
else:
|
||||
for col, corr in stats:
|
||||
if corr is None:
|
||||
continue
|
||||
# Size of existing BRIN index on this column, if any
|
||||
cr.execute(
|
||||
"""
|
||||
SELECT pg_size_pretty(pg_relation_size(idx.indexrelid))
|
||||
FROM pg_index idx
|
||||
JOIN pg_class i ON i.oid = idx.indexrelid
|
||||
JOIN pg_attribute a
|
||||
ON a.attrelid = %s::regclass
|
||||
AND a.attnum = ANY(idx.indkey)
|
||||
AND a.attname = %s
|
||||
JOIN pg_am am ON am.oid = i.relam AND am.amname = 'brin'
|
||||
WHERE idx.indrelid = %s::regclass
|
||||
LIMIT 1
|
||||
""",
|
||||
(table, col, table),
|
||||
)
|
||||
idx_row = cr.fetchone()
|
||||
idx_size = idx_row[0] if idx_row else "none"
|
||||
|
||||
label, color = recommend(corr)
|
||||
print(
|
||||
f" {col:<25} {corr:<14.4f} {idx_size:<12} {color}{label}{RESET}"
|
||||
)
|
||||
|
||||
print("─" * 70)
|
||||
|
||||
# ── Existing BRIN indexes ─────────────────────────────────────────────────────
|
||||
print(f"\n{BOLD}Existing BRIN indexes:{RESET}")
|
||||
cr.execute(
|
||||
"""
|
||||
SELECT indexname, indexdef
|
||||
FROM pg_indexes
|
||||
WHERE tablename = %s
|
||||
AND indexdef LIKE '%%USING brin%%'
|
||||
""",
|
||||
(table,),
|
||||
)
|
||||
brin_indexes = cr.fetchall()
|
||||
|
||||
if not brin_indexes:
|
||||
print(f" {YELLOW}No BRIN indexes found on this table.{RESET}")
|
||||
else:
|
||||
for iname, _ in brin_indexes:
|
||||
print(f" {GREEN}✔{RESET} {iname}")
|
||||
|
||||
print()
|
||||
286
script/shell_script_odoo/brin_cluster.py
Normal file
286
script/shell_script_odoo/brin_cluster.py
Normal file
|
|
@ -0,0 +1,286 @@
|
|||
#!/usr/bin/env python3
|
||||
"""
|
||||
brin_cluster.py — Reorders table rows physically to optimize BRIN indexes.
|
||||
|
||||
Strategy:
|
||||
1. Check correlation of candidate columns via pg_stats
|
||||
2. Pick the best column (highest absolute correlation, ideally create_date)
|
||||
3. Create a temporary B-tree index on that column
|
||||
4. CLUSTER the table on that index (rewrites rows in physical order)
|
||||
5. ANALYZE to refresh statistics
|
||||
6. Drop the temporary B-tree index
|
||||
7. Report the new correlation
|
||||
|
||||
WARNING:
|
||||
- CLUSTER acquires an ACCESS EXCLUSIVE lock — the table is fully locked.
|
||||
- On large tables, this can take minutes to hours.
|
||||
- Run during a maintenance window or off-peak hours.
|
||||
- Always pg_dump before running on production.
|
||||
|
||||
Usage:
|
||||
export MODEL=account.move.line
|
||||
cat brin_cluster.py | odoo-bin shell -d my_database --no-http
|
||||
|
||||
# Dry run (no changes, just shows what would be done):
|
||||
export MODEL=account.move.line DRY_RUN=1
|
||||
cat brin_cluster.py | odoo-bin shell -d my_database --no-http
|
||||
"""
|
||||
|
||||
import os
|
||||
import time
|
||||
|
||||
# ── Constants ─────────────────────────────────────────────────────────────────
|
||||
|
||||
# Columns considered as clustering candidates, in priority order.
|
||||
# create_date is first: it's never updated, so correlation is always perfect.
|
||||
CANDIDATE_COLUMNS = [
|
||||
"create_date",
|
||||
"date",
|
||||
"invoice_date",
|
||||
"write_date",
|
||||
"scheduled_date",
|
||||
"date_done",
|
||||
"in_date",
|
||||
]
|
||||
|
||||
BRIN_IDEAL_THRESHOLD = 0.9
|
||||
|
||||
RESET = "\033[0m"
|
||||
BOLD = "\033[1m"
|
||||
CYAN = "\033[0;36m"
|
||||
GREEN = "\033[0;32m"
|
||||
YELLOW = "\033[1;33m"
|
||||
RED = "\033[0;31m"
|
||||
|
||||
|
||||
# ── Config ────────────────────────────────────────────────────────────────────
|
||||
|
||||
model_name = (
|
||||
globals().get("MODEL") or os.environ.get("MODEL") or "account.move.line"
|
||||
)
|
||||
dry_run = bool(globals().get("DRY_RUN") or os.environ.get("DRY_RUN"))
|
||||
|
||||
if model_name not in env:
|
||||
table = model_name.replace(".", "_")
|
||||
else:
|
||||
table = env[model_name]._table
|
||||
|
||||
cr = env.cr
|
||||
cnx = env.cr._cnx
|
||||
|
||||
print(f"\n{BOLD}{CYAN}=== BRIN Cluster for: {model_name} ==={RESET}")
|
||||
print(f" Table : {table}")
|
||||
print(f" Dry run : {dry_run}\n")
|
||||
|
||||
|
||||
# ── Helpers ───────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def autocommit(sql, params=None):
|
||||
"""Execute a DDL statement outside the current transaction (AUTOCOMMIT)."""
|
||||
old = cnx.isolation_level
|
||||
cnx.set_isolation_level(0)
|
||||
with cnx.cursor() as c:
|
||||
c.execute(sql, params)
|
||||
cnx.set_isolation_level(old)
|
||||
|
||||
|
||||
def get_correlations():
|
||||
cr.execute(
|
||||
"""
|
||||
SELECT attname, correlation
|
||||
FROM pg_stats
|
||||
WHERE tablename = %s
|
||||
AND attname = ANY(%s)
|
||||
AND correlation IS NOT NULL
|
||||
ORDER BY ABS(correlation) DESC
|
||||
""",
|
||||
(table, CANDIDATE_COLUMNS),
|
||||
)
|
||||
return {row[0]: row[1] for row in cr.fetchall()}
|
||||
|
||||
|
||||
def table_size():
|
||||
cr.execute(
|
||||
"""
|
||||
SELECT
|
||||
pg_size_pretty(pg_total_relation_size(%s::regclass)),
|
||||
reltuples::bigint
|
||||
FROM pg_class WHERE relname = %s
|
||||
""",
|
||||
(table, table),
|
||||
)
|
||||
return cr.fetchone()
|
||||
|
||||
|
||||
def index_exists(index_name):
|
||||
cr.execute(
|
||||
"""
|
||||
SELECT COUNT(*) FROM pg_indexes
|
||||
WHERE tablename = %s AND indexname = %s
|
||||
""",
|
||||
(table, index_name),
|
||||
)
|
||||
return cr.fetchone()[0] > 0
|
||||
|
||||
|
||||
# ── Step 1: ANALYZE to get fresh stats ───────────────────────────────────────
|
||||
|
||||
print(f" {YELLOW}⟳ ANALYZE {table}...{RESET}", end=" ", flush=True)
|
||||
autocommit(f"ANALYZE {table}")
|
||||
print(f"{GREEN}OK{RESET}\n")
|
||||
|
||||
size_pretty, row_count = table_size()
|
||||
print(
|
||||
f" Size : {BOLD}{size_pretty}{RESET} (~{row_count:,} estimated rows)\n"
|
||||
)
|
||||
|
||||
# ── Step 2: Check current correlations ───────────────────────────────────────
|
||||
|
||||
correlations = get_correlations()
|
||||
|
||||
if not correlations:
|
||||
print(f"{RED}❌ No date columns found on {table}. Aborting.{RESET}\n")
|
||||
raise SystemExit(1)
|
||||
|
||||
print(f"{BOLD}Current correlations:{RESET}")
|
||||
for col, corr in sorted(correlations.items(), key=lambda x: -abs(x[1])):
|
||||
if abs(corr) >= BRIN_IDEAL_THRESHOLD:
|
||||
icon = f"{GREEN}✅ BRIN ideal{RESET}"
|
||||
else:
|
||||
icon = f"{YELLOW}⚠️ needs clustering{RESET}"
|
||||
print(f" {col:<25} {corr:+.4f} {icon}")
|
||||
|
||||
# ── Step 3: Pick the best clustering column ───────────────────────────────────
|
||||
|
||||
# Only cluster if at least one *candidate* column is suboptimal.
|
||||
# Note: a column like `date` (user-editable) may never reach ideal correlation
|
||||
# regardless of clustering — that's expected and not a reason to cluster.
|
||||
suboptimal = {
|
||||
col: corr
|
||||
for col, corr in correlations.items()
|
||||
if abs(corr) < BRIN_IDEAL_THRESHOLD
|
||||
}
|
||||
ideal = {
|
||||
col: corr
|
||||
for col, corr in correlations.items()
|
||||
if abs(corr) >= BRIN_IDEAL_THRESHOLD
|
||||
}
|
||||
|
||||
if suboptimal:
|
||||
print(f"\n {YELLOW}⚠️ Suboptimal columns: {', '.join(suboptimal)}{RESET}")
|
||||
print(
|
||||
f" {CYAN}Note: columns like 'date' or 'invoice_date' may stay suboptimal"
|
||||
)
|
||||
print(
|
||||
f" after clustering if users enter backdated records — that's expected.{RESET}"
|
||||
)
|
||||
else:
|
||||
print(
|
||||
f"\n{GREEN}✅ All columns already have ideal correlation. No clustering needed.{RESET}\n"
|
||||
)
|
||||
raise SystemExit(0)
|
||||
|
||||
# Pick the best clustering column: highest absolute correlation among candidates.
|
||||
# Prefer create_date first (never updated → guaranteed monotonic order),
|
||||
# then fall back to the column with highest abs(correlation).
|
||||
cluster_col = None
|
||||
for preferred in CANDIDATE_COLUMNS:
|
||||
if preferred in ideal: # already ideal → good anchor for physical order
|
||||
cluster_col = preferred
|
||||
break
|
||||
if not cluster_col:
|
||||
# All candidates are suboptimal — pick the least bad one
|
||||
cluster_col = max(suboptimal, key=lambda c: abs(suboptimal[c]))
|
||||
|
||||
if not cluster_col:
|
||||
print(f"{RED}❌ No suitable clustering column found. Aborting.{RESET}\n")
|
||||
raise SystemExit(1)
|
||||
|
||||
print(
|
||||
f"\n {BOLD}Clustering column chosen:{RESET} {cluster_col} "
|
||||
f"(current correlation: {correlations[cluster_col]:+.4f})\n"
|
||||
)
|
||||
|
||||
# ── Step 4: Create temporary B-tree index ────────────────────────────────────
|
||||
|
||||
tmp_index = f"_brin_cluster_tmp_{table}_{cluster_col}"
|
||||
|
||||
if index_exists(tmp_index):
|
||||
print(
|
||||
f" {YELLOW}⚠️ Temporary index already exists, dropping it first...{RESET}"
|
||||
)
|
||||
if not dry_run:
|
||||
autocommit(f'DROP INDEX IF EXISTS "{tmp_index}"')
|
||||
|
||||
print(
|
||||
f" {YELLOW}⟳ Creating temporary B-tree index on {cluster_col}...{RESET}",
|
||||
end=" ",
|
||||
flush=True,
|
||||
)
|
||||
if not dry_run:
|
||||
autocommit(f'CREATE INDEX "{tmp_index}" ON "{table}" ("{cluster_col}")')
|
||||
print(f"{GREEN}OK{RESET}")
|
||||
else:
|
||||
print(f"{CYAN}[DRY RUN]{RESET}")
|
||||
|
||||
# ── Step 5: CLUSTER ───────────────────────────────────────────────────────────
|
||||
|
||||
print(f"\n {YELLOW}⟳ CLUSTER {table} on {cluster_col}...{RESET}")
|
||||
print(
|
||||
f" {YELLOW} (ACCESS EXCLUSIVE lock — table unavailable during this operation){RESET}",
|
||||
end=" ",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
if not dry_run:
|
||||
t0 = time.time()
|
||||
autocommit(f'CLUSTER "{table}" USING "{tmp_index}"')
|
||||
elapsed = time.time() - t0
|
||||
print(f"{GREEN}OK{RESET} ({elapsed:.1f}s)")
|
||||
else:
|
||||
print(f"{CYAN}[DRY RUN]{RESET}")
|
||||
|
||||
# ── Step 6: Drop temporary index ─────────────────────────────────────────────
|
||||
|
||||
print(
|
||||
f"\n {YELLOW}⟳ Dropping temporary index...{RESET}", end=" ", flush=True
|
||||
)
|
||||
if not dry_run:
|
||||
autocommit(f'DROP INDEX IF EXISTS "{tmp_index}"')
|
||||
print(f"{GREEN}OK{RESET}")
|
||||
else:
|
||||
print(f"{CYAN}[DRY RUN]{RESET}")
|
||||
|
||||
# ── Step 7: ANALYZE again + report new correlations ──────────────────────────
|
||||
|
||||
print(
|
||||
f"\n {YELLOW}⟳ ANALYZE after clustering...{RESET}", end=" ", flush=True
|
||||
)
|
||||
if not dry_run:
|
||||
autocommit(f"ANALYZE {table}")
|
||||
print(f"{GREEN}OK{RESET}\n")
|
||||
|
||||
new_correlations = get_correlations()
|
||||
|
||||
print(f"{BOLD}Correlations after clustering:{RESET}")
|
||||
print("─" * 55)
|
||||
print(f" {'COLUMN':<25} {'BEFORE':>8} {'AFTER':>8} RESULT")
|
||||
print("─" * 55)
|
||||
for col in sorted(correlations.keys()):
|
||||
before = correlations.get(col, 0.0)
|
||||
after = new_correlations.get(col, 0.0)
|
||||
delta = after - before
|
||||
if abs(after) >= BRIN_IDEAL_THRESHOLD:
|
||||
result = f"{GREEN}✅ BRIN ideal{RESET}"
|
||||
else:
|
||||
result = f"{YELLOW}⚠️ still suboptimal{RESET}"
|
||||
arrow = "↑" if delta > 0.01 else ("↓" if delta < -0.01 else "→")
|
||||
print(
|
||||
f" {col:<25} {before:>+8.4f} {after:>+8.4f} {arrow} {result}"
|
||||
)
|
||||
print("─" * 55)
|
||||
else:
|
||||
print(f"{CYAN}[DRY RUN]{RESET}")
|
||||
|
||||
print(f"\n{GREEN}✅ Done.{RESET}\n")
|
||||
Loading…
Reference in a new issue