Daniel Allaire
49ceb81a4d
Les hyperviseurs exposaient temperatures, ventilateurs, SMART et usure NVMe sans que rien les regarde. scripts/materiel.py porte capteurs et seuils ; Grafana en tire le tableau Materiel, Icinga un service materiel par hyperviseur. Premier verdict : le disque sda de gandalf (OSD Ceph) a 56 secteurs en attente, stables. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
206 lines
13 KiB
Django/Jinja
206 lines
13 KiB
Django/Jinja
{#- GENERE par Set-OPS (role serveur_grafana). Ne pas editer a la main.
|
|
|
|
LE TABLEAU DU MATERIEL — hyperviseurs et frontiere (2026-09-17).
|
|
|
|
Assemble depuis `scripts/materiel.py`, le MEME catalogue que la sonde Icinga lit : un
|
|
seuil change la-bas change la couleur ici et le verdict la-bas. Deux ecritures
|
|
divergeraient, et un graphe rouge cohabiterait avec une supervision verte.
|
|
|
|
Le JSON est construit comme une structure Jinja puis rendu d'un bloc par `tojson` :
|
|
aucune virgule a la main, donc aucun tableau que Grafana sauterait en silence.
|
|
-#}
|
|
{%- set cat = setops_materiel -%}
|
|
{%- set filtre = ' and on(hote) up{job=~"fabric|frontiere",hote=~"$hote"}' -%}
|
|
{%- set panneaux = [] -%}
|
|
{%- set y = namespace(v=0, id=0) -%}
|
|
|
|
{%- macro seuils(f) -%}
|
|
{{ {"mode": "absolute", "steps": [
|
|
{"color": "green", "value": none},
|
|
{"color": "orange", "value": f.avert},
|
|
{"color": "red", "value": f.crit}]} | tojson }}
|
|
{%- endmacro -%}
|
|
|
|
{#- ---- RANGEE : ETAT EN DIRECT ------------------------------------------------------ #}
|
|
{%- set y.id = y.id + 1 -%}
|
|
{%- set _ = panneaux.append({"type": "row", "id": y.id, "title": "État en direct", "collapsed": false,
|
|
"gridPos": {"h": 1, "w": 24, "x": 0, "y": y.v}}) -%}
|
|
{%- set y.v = y.v + 1 -%}
|
|
{%- set tuiles = cat.temperatures | selectattr("cle", "in", ["processeur", "carte_mere", "nvme", "disques"]) | list -%}
|
|
{%- for f in tuiles -%}
|
|
{%- set y.id = y.id + 1 -%}
|
|
{%- set _ = panneaux.append({
|
|
"type": "stat", "id": y.id,
|
|
"title": f.titre ~ " — maximum",
|
|
"description": f.raison,
|
|
"datasource": {"type": "prometheus", "uid": "${ds_prom}"},
|
|
"gridPos": {"h": 5, "w": 6, "x": loop.index0 * 6, "y": y.v},
|
|
"fieldConfig": {"defaults": {"unit": "celsius", "decimals": 0,
|
|
"thresholds": (seuils(f) | from_json)}, "overrides": []},
|
|
"options": {"colorMode": "background", "graphMode": "area", "justifyMode": "center",
|
|
"textMode": "value_and_name", "orientation": "vertical",
|
|
"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}},
|
|
"targets": [{"refId": "A", "datasource": {"type": "prometheus", "uid": "${ds_prom}"},
|
|
"expr": "max by (hote) ((" ~ f.expr_juge ~ ")" ~ filtre ~ ")",
|
|
"legendFormat": "{{hote}}"}]}) -%}
|
|
{%- endfor -%}
|
|
{%- set y.v = y.v + 5 -%}
|
|
|
|
{#- Les controles : une tuile chacun. Zero = vert ; sinon la couleur de SON niveau. #}
|
|
{%- set largeur = [24 // (cat.controles | length), 3] | max -%}
|
|
{%- for c in cat.controles -%}
|
|
{%- set y.id = y.id + 1 -%}
|
|
{%- set _ = panneaux.append({
|
|
"type": "stat", "id": y.id,
|
|
"title": c.titre,
|
|
"description": c.raison,
|
|
"datasource": {"type": "prometheus", "uid": "${ds_prom}"},
|
|
"gridPos": {"h": 4, "w": largeur, "x": (loop.index0 * largeur) % 24, "y": y.v + ((loop.index0 * largeur) // 24) * 4},
|
|
"fieldConfig": {"defaults": {"unit": "none", "decimals": 0,
|
|
"noValue": "0",
|
|
"thresholds": {"mode": "absolute", "steps": [
|
|
{"color": "green", "value": none},
|
|
{"color": ("red" if c.niveau == "crit" else "orange"), "value": 1}]}},
|
|
"overrides": []},
|
|
"options": {"colorMode": "background", "graphMode": "none", "justifyMode": "center",
|
|
"textMode": "value", "orientation": "auto",
|
|
"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}},
|
|
"targets": [{"refId": "A", "datasource": {"type": "prometheus", "uid": "${ds_prom}"},
|
|
"expr": "count(count by (hote, capteur) ((" ~ c.expr ~ ")" ~ filtre ~ ")) or vector(0)"}]}) -%}
|
|
{%- endfor -%}
|
|
{%- set y.v = y.v + (((cat.controles | length) * largeur + 23) // 24) * 4 -%}
|
|
|
|
{#- ---- RANGEE : TEMPERATURES -------------------------------------------------------- #}
|
|
{%- set y.id = y.id + 1 -%}
|
|
{%- set _ = panneaux.append({"type": "row", "id": y.id, "title": "Températures", "collapsed": false,
|
|
"gridPos": {"h": 1, "w": 24, "x": 0, "y": y.v}}) -%}
|
|
{%- set y.v = y.v + 1 -%}
|
|
{%- for f in cat.temperatures -%}
|
|
{%- set y.id = y.id + 1 -%}
|
|
{%- set _ = panneaux.append({
|
|
"type": "timeseries", "id": y.id,
|
|
"title": f.titre,
|
|
"description": f.raison ~ " — Avertissement " ~ f.avert ~ " °C, critique " ~ f.crit ~ " °C (verdict Icinga lisse sur " ~ f.jugee_sur ~ ").",
|
|
"datasource": {"type": "prometheus", "uid": "${ds_prom}"},
|
|
"gridPos": {"h": 9, "w": 12, "x": 0 if loop.index0 % 2 == 0 else 12, "y": y.v + (loop.index0 // 2) * 9},
|
|
"fieldConfig": {"defaults": {
|
|
"unit": "celsius", "decimals": 1,
|
|
"color": {"mode": "palette-classic"},
|
|
"thresholds": (seuils(f) | from_json),
|
|
"custom": {"drawStyle": "line", "lineWidth": 2, "fillOpacity": 8, "gradientMode": "opacity",
|
|
"showPoints": "never", "spanNulls": true,
|
|
"thresholdsStyle": {"mode": "dashed"}}},
|
|
"overrides": []},
|
|
"options": {"legend": {"displayMode": "table", "placement": "bottom", "calcs": ["lastNotNull", "max"]},
|
|
"tooltip": {"mode": "multi", "sort": "desc"}},
|
|
"targets": [{"refId": "A", "datasource": {"type": "prometheus", "uid": "${ds_prom}"},
|
|
"expr": "(" ~ f.expr ~ ")" ~ filtre,
|
|
"legendFormat": "{{hote}} · {{capteur}}"}]}) -%}
|
|
{%- endfor -%}
|
|
{%- set y.v = y.v + (((cat.temperatures | length) + 1) // 2) * 9 -%}
|
|
|
|
{#- ---- RANGEE : VENTILATEURS ET PUISSANCE ----------------------------------------------- #}
|
|
{%- set y.id = y.id + 1 -%}
|
|
{%- set _ = panneaux.append({"type": "row", "id": y.id, "title": "Ventilateurs et puissance", "collapsed": false,
|
|
"gridPos": {"h": 1, "w": 24, "x": 0, "y": y.v}}) -%}
|
|
{%- set y.v = y.v + 1 -%}
|
|
{%- set y.id = y.id + 1 -%}
|
|
{%- set _ = panneaux.append({
|
|
"type": "timeseries", "id": y.id, "title": "Ventilateurs",
|
|
"description": "Seuls les connecteurs qui ont tourne dans les dernieres 24 h : un connecteur vide lit 0 depuis toujours et n'apprend rien.",
|
|
"datasource": {"type": "prometheus", "uid": "${ds_prom}"},
|
|
"gridPos": {"h": 9, "w": 12, "x": 0, "y": y.v},
|
|
"fieldConfig": {"defaults": {"unit": "rotrpm", "decimals": 0, "color": {"mode": "palette-classic"},
|
|
"custom": {"drawStyle": "line", "lineWidth": 2, "fillOpacity": 5, "showPoints": "never", "spanNulls": true}},
|
|
"overrides": []},
|
|
"options": {"legend": {"displayMode": "table", "placement": "bottom", "calcs": ["lastNotNull", "min"]},
|
|
"tooltip": {"mode": "multi", "sort": "desc"}},
|
|
"targets": [{"refId": "A", "datasource": {"type": "prometheus", "uid": "${ds_prom}"},
|
|
"expr": "(node_hwmon_fan_rpm{job=~\"fabric|frontiere\"} and on(instance, chip, sensor) (max_over_time(node_hwmon_fan_rpm{job=~\"fabric|frontiere\"}[1d]) > 0))" ~ filtre,
|
|
"legendFormat": "{{hote}} · {{sensor}}"}]}) -%}
|
|
{%- set y.id = y.id + 1 -%}
|
|
{%- set _ = panneaux.append({
|
|
"type": "timeseries", "id": y.id, "title": "Puissance du processeur (PPT)",
|
|
"description": "Puissance du boitier processeur annoncee par le pilote amdgpu (APU integre). Elle suit la charge : une temperature qui monte SANS puissance qui monte parle du refroidissement.",
|
|
"datasource": {"type": "prometheus", "uid": "${ds_prom}"},
|
|
"gridPos": {"h": 9, "w": 12, "x": 12, "y": y.v},
|
|
"fieldConfig": {"defaults": {"unit": "watt", "decimals": 1, "color": {"mode": "palette-classic"},
|
|
"custom": {"drawStyle": "line", "lineWidth": 2, "fillOpacity": 15, "gradientMode": "opacity", "showPoints": "never", "spanNulls": true}},
|
|
"overrides": []},
|
|
"options": {"legend": {"displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"]},
|
|
"tooltip": {"mode": "multi", "sort": "desc"}},
|
|
"targets": [{"refId": "A", "datasource": {"type": "prometheus", "uid": "${ds_prom}"},
|
|
"expr": "(node_hwmon_power_watt{job=~\"fabric|frontiere\"} * on(instance, chip, sensor) group_left(label) group by (instance, chip, sensor, label) (node_hwmon_sensor_label{job=~\"fabric|frontiere\",label=\"PPT\"}))" ~ filtre,
|
|
"legendFormat": "{{hote}}"}]}) -%}
|
|
{%- set y.v = y.v + 9 -%}
|
|
|
|
{#- ---- RANGEE : STOCKAGE ------------------------------------------------------------ #}
|
|
{%- set y.id = y.id + 1 -%}
|
|
{%- set _ = panneaux.append({"type": "row", "id": y.id, "title": "Stockage", "collapsed": false,
|
|
"gridPos": {"h": 1, "w": 24, "x": 0, "y": y.v}}) -%}
|
|
{%- set y.v = y.v + 1 -%}
|
|
{%- set y.id = y.id + 1 -%}
|
|
{%- set _ = panneaux.append({
|
|
"type": "bargauge", "id": y.id, "title": "Usure des NVMe",
|
|
"description": "Part de l'endurance annoncee deja consommee. Avertissement a 80 % : a commander, pas encore a remplacer.",
|
|
"datasource": {"type": "prometheus", "uid": "${ds_prom}"},
|
|
"gridPos": {"h": 9, "w": 8, "x": 0, "y": y.v},
|
|
"fieldConfig": {"defaults": {"unit": "percentunit", "min": 0, "max": 1, "decimals": 0,
|
|
"thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": none}, {"color": "orange", "value": 0.8}, {"color": "red", "value": 0.9}]}},
|
|
"overrides": []},
|
|
"options": {"displayMode": "gradient", "orientation": "horizontal", "showUnfilled": true, "valueMode": "color",
|
|
"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}},
|
|
"targets": [{"refId": "A", "datasource": {"type": "prometheus", "uid": "${ds_prom}"},
|
|
"expr": "nvme_percentage_used_ratio{job=~\"fabric|frontiere\"}" ~ filtre,
|
|
"legendFormat": "{{hote}} · {{device}}"}]}) -%}
|
|
{%- set y.id = y.id + 1 -%}
|
|
{%- set _ = panneaux.append({
|
|
"type": "bargauge", "id": y.id, "title": "Secteurs défaillants",
|
|
"description": "En attente + irreparables + realloues, par disque SATA. Un compte stable se surveille ; un compte qui MONTE se remplace (controle « secteurs en progression »).",
|
|
"datasource": {"type": "prometheus", "uid": "${ds_prom}"},
|
|
"gridPos": {"h": 9, "w": 8, "x": 8, "y": y.v},
|
|
"fieldConfig": {"defaults": {"unit": "none", "min": 0, "decimals": 0,
|
|
"thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": none}, {"color": "orange", "value": 1}, {"color": "red", "value": 100}]}},
|
|
"overrides": []},
|
|
"options": {"displayMode": "gradient", "orientation": "horizontal", "showUnfilled": true, "valueMode": "color",
|
|
"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}},
|
|
"targets": [{"refId": "A", "datasource": {"type": "prometheus", "uid": "${ds_prom}"},
|
|
"expr": "(sum by (hote, disk) (smartmon_current_pending_sector_raw_value{job=~\"fabric|frontiere\"} or smartmon_offline_uncorrectable_raw_value{job=~\"fabric|frontiere\"} or smartmon_reallocated_sector_ct_raw_value{job=~\"fabric|frontiere\"}))" ~ filtre,
|
|
"legendFormat": "{{hote}} · {{disk}}"}]}) -%}
|
|
{%- set y.id = y.id + 1 -%}
|
|
{%- set _ = panneaux.append({
|
|
"type": "bargauge", "id": y.id, "title": "Heures de fonctionnement",
|
|
"description": "Age reel des disques. Un disque mecanique au-dela de 40 000 heures (4,5 ans) entre dans la zone ou les pannes se multiplient.",
|
|
"datasource": {"type": "prometheus", "uid": "${ds_prom}"},
|
|
"gridPos": {"h": 9, "w": 8, "x": 16, "y": y.v},
|
|
"fieldConfig": {"defaults": {"unit": "h", "min": 0, "decimals": 0,
|
|
"thresholds": {"mode": "absolute", "steps": [{"color": "blue", "value": none}, {"color": "orange", "value": 40000}]}},
|
|
"overrides": []},
|
|
"options": {"displayMode": "gradient", "orientation": "horizontal", "showUnfilled": true, "valueMode": "color",
|
|
"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}},
|
|
"targets": [{"refId": "A", "datasource": {"type": "prometheus", "uid": "${ds_prom}"},
|
|
"expr": "(label_replace(smartmon_power_on_hours_raw_value{job=~\"fabric|frontiere\",type=\"sat\"}, \"dispositif\", \"$1\", \"disk\", \"/dev/(.*)\") or label_replace(nvme_power_on_hours_total{job=~\"fabric|frontiere\"}, \"dispositif\", \"$1\", \"device\", \"(.*)\"))" ~ filtre,
|
|
"legendFormat": "{{hote}} · {{dispositif}}"}]}) -%}
|
|
|
|
{{ {
|
|
"uid": "setops-materiel",
|
|
"title": "Set-OPS — Matériel",
|
|
"description": "Hyperviseurs et frontiere : temperatures, ventilateurs, puissance, disques. Meme catalogue et memes seuils que le service Icinga « materiel » (scripts/materiel.py).",
|
|
"tags": ["set-ops", "derive", "materiel", "fabric"],
|
|
"timezone": "browser",
|
|
"refresh": "1m",
|
|
"time": {"from": "now-24h", "to": "now"},
|
|
"schemaVersion": 39,
|
|
"version": 1,
|
|
"graphTooltip": 1,
|
|
"templating": {"list": [
|
|
{"name": "ds_prom", "label": "Source Prometheus", "type": "datasource", "query": "prometheus", "hide": 2, "current": {}},
|
|
{"name": "hote", "label": "Équipement", "type": "query",
|
|
"datasource": {"type": "prometheus", "uid": "${ds_prom}"},
|
|
"query": {"query": "label_values(up{job=~\"fabric|frontiere\"}, hote)", "refId": "hote"},
|
|
"definition": "label_values(up{job=~\"fabric|frontiere\"}, hote)",
|
|
"multi": true, "includeAll": true, "current": {"selected": true, "text": ["All"], "value": ["$__all"]},
|
|
"refresh": 2, "sort": 1}
|
|
]},
|
|
"panels": panneaux
|
|
} | tojson }}
|