86 lines
3.3 KiB
YAML
86 lines
3.3 KiB
YAML
apiVersion: monitoring.coreos.com/v1
|
|
kind: PrometheusRule
|
|
metadata:
|
|
name: im-server-p0-p1
|
|
namespace: im-server-system
|
|
spec:
|
|
groups:
|
|
- name: im-server-cluster
|
|
rules:
|
|
- alert: IMClusterP2MQueueHigh
|
|
expr: im_cluster_p2m_queue_length / clamp_min(im_cluster_p2m_queue_capacity, 1) > 0.70
|
|
for: 30s
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "IM cluster proxy queue is above 70%"
|
|
- alert: IMClusterP2MQueueCritical
|
|
expr: im_cluster_p2m_queue_length / clamp_min(im_cluster_p2m_queue_capacity, 1) > 0.90
|
|
for: 30s
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: "IM cluster proxy queue is above 90%"
|
|
- alert: IMClusterEnqueueTimeout
|
|
expr: sum by (pod, peer) (increase(im_cluster_p2m_enqueue_total{result="timeout"}[1m])) > 0
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: "IM cluster rejected proxy-to-master enqueue"
|
|
- alert: IMClusterRouteRejected
|
|
expr: sum by (pod, peer, result) (increase(im_cluster_route_total{result=~"hub_full|peer_rejected"}[1m])) > 0
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: "IM cluster route was rejected"
|
|
- alert: IMClusterPeerUnavailable
|
|
expr: im_cluster_live_peers < im_cluster_expected_peers
|
|
for: 15s
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: "IM server has unavailable cluster peers"
|
|
- alert: IMWebSocketConnectionsImbalanced
|
|
expr: max(im_ws_connections_active) / clamp_min(avg(im_ws_connections_active), 1) > 1.30
|
|
for: 2m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "IM WebSocket connections are imbalanced across pods"
|
|
- name: im-server-persistence
|
|
rules:
|
|
- alert: IMDatabasePoolHigh
|
|
expr: im_db_pool_in_use_connections / clamp_min(im_db_pool_max_open_connections, 1) > 0.85
|
|
for: 1m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "IM database pool usage is above 85%"
|
|
- alert: IMDatabasePoolCritical
|
|
expr: im_db_pool_in_use_connections / clamp_min(im_db_pool_max_open_connections, 1) > 0.95
|
|
for: 30s
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: "IM database pool usage is above 95%"
|
|
- alert: IMKafkaPersistDeadLetter
|
|
expr: sum by (kind, reason) (increase(im_kafka_dead_letters_total[5m])) > 0
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: "IM Kafka persistence produced dead letters"
|
|
- alert: IMKafkaConsumerLag
|
|
expr: max by (pod, topic, group) (im_kafka_consumer_lag) > 1000
|
|
for: 2m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "IM Kafka consumer lag is above 1000"
|
|
- alert: IMKafkaBacklogOld
|
|
expr: max by (pod, topic, group) (im_kafka_backlog_oldest_age_seconds) > 60
|
|
for: 2m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: "IM Kafka persistence backlog is older than 60 seconds"
|