fix: harden multi-pod delivery and load verification

monitoring/README.md

@@ -5,3 +5,5 @@ This directory contains code related to monitoring Tinode server. Supported moni
 * [InfluxDB](https://www.influxdata.com/)
 
 See [exporter/README](./exporter/README.md) for more details.
+
+`im-server-alerts.yaml` contains the P0/P1 cluster routing, WebSocket balance, Kafka lag, backlog age, and dead-letter alert rules for a Prometheus Operator deployment. Apply it only after the cluster has the `PrometheusRule` CRD installed.

monitoring/im-server-alerts.yaml

@@ -0,0 +1,71 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: im-server-p0-p1
+  namespace: im-server-system
+spec:
+  groups:
+    - name: im-server-cluster
+      rules:
+        - alert: IMClusterP2MQueueHigh
+          expr: im_cluster_p2m_queue_length / clamp_min(im_cluster_p2m_queue_capacity, 1) > 0.70
+          for: 30s
+          labels:
+            severity: warning
+          annotations:
+            summary: "IM cluster proxy queue is above 70%"
+        - alert: IMClusterP2MQueueCritical
+          expr: im_cluster_p2m_queue_length / clamp_min(im_cluster_p2m_queue_capacity, 1) > 0.90
+          for: 30s
+          labels:
+            severity: critical
+          annotations:
+            summary: "IM cluster proxy queue is above 90%"
+        - alert: IMClusterEnqueueTimeout
+          expr: sum by (pod, peer) (increase(im_cluster_p2m_enqueue_total{result="timeout"}[1m])) > 0
+          labels:
+            severity: critical
+          annotations:
+            summary: "IM cluster rejected proxy-to-master enqueue"
+        - alert: IMClusterRouteRejected
+          expr: sum by (pod, peer, result) (increase(im_cluster_route_total{result=~"hub_full|peer_rejected"}[1m])) > 0
+          labels:
+            severity: critical
+          annotations:
+            summary: "IM cluster route was rejected"
+        - alert: IMClusterPeerUnavailable
+          expr: im_cluster_live_peers < im_cluster_expected_peers
+          for: 15s
+          labels:
+            severity: critical
+          annotations:
+            summary: "IM server has unavailable cluster peers"
+        - alert: IMWebSocketConnectionsImbalanced
+          expr: max(im_ws_connections_active) / clamp_min(avg(im_ws_connections_active), 1) > 1.30
+          for: 2m
+          labels:
+            severity: warning
+          annotations:
+            summary: "IM WebSocket connections are imbalanced across pods"
+    - name: im-server-persistence
+      rules:
+        - alert: IMKafkaPersistDeadLetter
+          expr: sum by (kind, reason) (increase(im_kafka_dead_letters_total[5m])) > 0
+          labels:
+            severity: critical
+          annotations:
+            summary: "IM Kafka persistence produced dead letters"
+        - alert: IMKafkaConsumerLag
+          expr: max by (pod, topic, group) (im_kafka_consumer_lag) > 1000
+          for: 2m
+          labels:
+            severity: warning
+          annotations:
+            summary: "IM Kafka consumer lag is above 1000"
+        - alert: IMKafkaBacklogOld
+          expr: max by (pod, topic, group) (im_kafka_backlog_oldest_age_seconds) > 60
+          for: 2m
+          labels:
+            severity: critical
+          annotations:
+            summary: "IM Kafka persistence backlog is older than 60 seconds"