apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: im-server-p0-p1
  namespace: im-server-system
spec:
  groups:
    - name: im-server-cluster
      rules:
        - alert: IMClusterP2MQueueHigh
          expr: im_cluster_p2m_queue_length / clamp_min(im_cluster_p2m_queue_capacity, 1) > 0.70
          for: 30s
          labels:
            severity: warning
          annotations:
            summary: "IM cluster proxy queue is above 70%"
        - alert: IMClusterP2MQueueCritical
          expr: im_cluster_p2m_queue_length / clamp_min(im_cluster_p2m_queue_capacity, 1) > 0.90
          for: 30s
          labels:
            severity: critical
          annotations:
            summary: "IM cluster proxy queue is above 90%"
        - alert: IMClusterEnqueueTimeout
          expr: sum by (pod, peer) (increase(im_cluster_p2m_enqueue_total{result="timeout"}[1m])) > 0
          labels:
            severity: critical
          annotations:
            summary: "IM cluster rejected proxy-to-master enqueue"
        - alert: IMClusterRouteRejected
          expr: sum by (pod, peer, result) (increase(im_cluster_route_total{result=~"hub_full|peer_rejected"}[1m])) > 0
          labels:
            severity: critical
          annotations:
            summary: "IM cluster route was rejected"
        - alert: IMClusterPeerUnavailable
          expr: im_cluster_live_peers < im_cluster_expected_peers
          for: 15s
          labels:
            severity: critical
          annotations:
            summary: "IM server has unavailable cluster peers"
        - alert: IMWebSocketConnectionsImbalanced
          expr: max(im_ws_connections_active) / clamp_min(avg(im_ws_connections_active), 1) > 1.30
          for: 2m
          labels:
            severity: warning
          annotations:
            summary: "IM WebSocket connections are imbalanced across pods"
    - name: im-server-persistence
      rules:
        - alert: IMDatabasePoolHigh
          expr: im_db_pool_in_use_connections / clamp_min(im_db_pool_max_open_connections, 1) > 0.85
          for: 1m
          labels:
            severity: warning
          annotations:
            summary: "IM database pool usage is above 85%"
        - alert: IMDatabasePoolCritical
          expr: im_db_pool_in_use_connections / clamp_min(im_db_pool_max_open_connections, 1) > 0.95
          for: 30s
          labels:
            severity: critical
          annotations:
            summary: "IM database pool usage is above 95%"
        - alert: IMKafkaPersistDeadLetter
          expr: sum by (kind, reason) (increase(im_kafka_dead_letters_total[5m])) > 0
          labels:
            severity: critical
          annotations:
            summary: "IM Kafka persistence produced dead letters"
        - alert: IMKafkaConsumerLag
          expr: max by (pod, topic, group) (im_kafka_consumer_lag) > 1000
          for: 2m
          labels:
            severity: warning
          annotations:
            summary: "IM Kafka consumer lag is above 1000"
        - alert: IMKafkaBacklogOld
          expr: max by (pod, topic, group) (im_kafka_backlog_oldest_age_seconds) > 60
          for: 2m
          labels:
            severity: critical
          annotations:
            summary: "IM Kafka persistence backlog is older than 60 seconds"