###
# Sample prometheus rules/alerts for mysqld.
#
# NOTE: Please review these carefully as thresholds and behavior may not meet
#       your SLOs or labels.
#
###

groups:
- name: MySQLdAlerts
  rules:
  - alert: MySQLDown
    annotations:
      description: MySQL {{$labels.job}} on {{$labels.instance}} is not up.
      summary: MySQL not up
    expr: mysql_up != 1
    for: 5m
    labels:
      severity: critical
- name: GaleraAlerts
  rules:
  - alert: MySQLGaleraNotReady
    annotations:
      description: '{{$labels.job}} on {{$labels.instance}} is not ready.'
      summary: Galera cluster node not ready
    expr: mysql_global_status_wsrep_ready != 1
    for: 5m
    labels:
      severity: warning
  - alert: MySQLGaleraOutOfSync
    annotations:
      description: '{{$labels.job}} on {{$labels.instance}} is not in sync ({{$value}} != 4).'
      summary: Galera cluster node out of sync
    expr: (mysql_global_status_wsrep_local_state != 4 and mysql_global_variables_wsrep_desync == 0)
    for: 5m
    labels:
      severity: warning
  - alert: MySQLGaleraDonorFallingBehind
    annotations:
      description: '{{$labels.job}} on {{$labels.instance}} is a donor (hotbackup) and is falling behind (queue size {{$value}}).'
      summary: xtradb cluster donor node falling behind
    expr: (mysql_global_status_wsrep_local_state == 2 and mysql_global_status_wsrep_local_recv_queue > 100)
    for: 5m
    labels:
      severity: warning
  - alert: MySQLReplicationNotRunning
    annotations:
      description: Slave replication (IO or SQL) has been down for more than 2 minutes.
      summary: Slave replication is not running
    expr: mysql_slave_status_slave_io_running == 0 or mysql_slave_status_slave_sql_running == 0
    for: 2m
    labels:
      severity: critical
  - alert: MySQLReplicationLag
    annotations:
      description: The mysql slave replication has fallen behind and is not recovering
      summary: MySQL slave replication is lagging
    expr: (instance:mysql_slave_lag_seconds > 30) and on(instance) (predict_linear(instance:mysql_slave_lag_seconds[5m], 60 * 2) > 0)
    for: 1m
    labels:
      severity: critical
  - alert: MySQLReplicationLag
    annotations:
      description: The mysql slave replication has fallen behind and is not recovering
      summary: MySQL slave replication is lagging
    expr: (instance:mysql_heartbeat_lag_seconds > 30) and on(instance) (predict_linear(instance:mysql_heartbeat_lag_seconds[5m], 60 * 2) > 0)
    for: 1m
    labels:
      severity: critical
  - alert: MySQLInnoDBLogWaits
    annotations:
      description: The innodb logs are waiting for disk at a rate of {{$value}} / second
      summary: MySQL innodb log writes stalling
    expr: rate(mysql_global_status_innodb_log_waits[15m]) > 10
    labels:
      severity: warning
