fix: harden multi-pod delivery and load verification

server/metrics/metrics.go

@@ -2,7 +2,9 @@ package metrics
 
 import (
 	"expvar"
+	"os"
 	"strconv"
+	"strings"
 	"time"
 
 	"github.com/prometheus/client_golang/prometheus"
@@ -45,11 +47,98 @@ var (
 	}, func() float64 {
 		return float64(expvarIntValue("LiveSessions"))
 	})
+	IMWSConnectionsActive = prometheus.NewGaugeFunc(prometheus.GaugeOpts{
+		Name:        "im_ws_connections_active",
+		Help:        "Current active WebSocket connections, using the IM metric namespace.",
+		ConstLabels: prometheus.Labels{"pod": metricPodName()},
+	}, func() float64 {
+		return float64(expvarIntValue("LiveSessions"))
+	})
+	IMClusterLivePeers = prometheus.NewGaugeFunc(prometheus.GaugeOpts{
+		Name:        "im_cluster_live_peers",
+		Help:        "Current connected cluster peers.",
+		ConstLabels: prometheus.Labels{"pod": metricPodName()},
+	}, func() float64 {
+		return float64(expvarIntValue("LiveClusterNodes"))
+	})
+	IMClusterExpectedPeers = prometheus.NewGaugeFunc(prometheus.GaugeOpts{
+		Name:        "im_cluster_expected_peers",
+		Help:        "Expected cluster peers derived from configured total nodes.",
+		ConstLabels: prometheus.Labels{"pod": metricPodName()},
+	}, func() float64 {
+		total := expvarIntValue("TotalClusterNodes")
+		if total > 0 {
+			total--
+		}
+		return float64(total)
+	})
+
+	ClusterP2MQueueLength = prometheus.NewGaugeVec(prometheus.GaugeOpts{
+		Name: "im_cluster_p2m_queue_length",
+		Help: "Current proxy-to-master queue depth by pod and peer.",
+	}, []string{"pod", "peer"})
+	ClusterP2MQueueCapacity = prometheus.NewGaugeVec(prometheus.GaugeOpts{
+		Name: "im_cluster_p2m_queue_capacity",
+		Help: "Configured proxy-to-master queue capacity by pod and peer.",
+	}, []string{"pod", "peer"})
+	ClusterP2MEnqueueTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
+		Name: "im_cluster_p2m_enqueue_total",
+		Help: "Proxy-to-master enqueue attempts by pod, peer, and result.",
+	}, []string{"pod", "peer", "result"})
+	ClusterRPCInflight = prometheus.NewGaugeVec(prometheus.GaugeOpts{
+		Name: "im_cluster_rpc_inflight",
+		Help: "Current synchronous cluster RPC calls by pod, peer, and method.",
+	}, []string{"pod", "peer", "method"})
+	ClusterRPCDuration = prometheus.NewHistogramVec(prometheus.HistogramOpts{
+		Name:    "im_cluster_rpc_duration_seconds",
+		Help:    "Cluster RPC duration by pod, peer, method, and result.",
+		Buckets: []float64{0.001, 0.003, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5},
+	}, []string{"pod", "peer", "method", "result"})
+	ClusterRouteTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
+		Name: "im_cluster_route_total",
+		Help: "Cluster route attempts by pod, peer, and result.",
+	}, []string{"pod", "peer", "result"})
+	ClusterProxyResponseTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
+		Name: "im_cluster_proxy_response_total",
+		Help: "Cluster proxy responses by pod and result.",
+	}, []string{"pod", "result"})
+	MessagePersistDuration = prometheus.NewHistogramVec(prometheus.HistogramOpts{
+		Name:    "im_message_persist_duration_seconds",
+		Help:    "Kafka message persistence duration by message kind and result.",
+		Buckets: []float64{0.001, 0.003, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10},
+	}, []string{"kind", "result"})
+	KafkaMessagePersistTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
+		Name: "im_kafka_message_persist_total",
+		Help: "Kafka message persistence outcomes by message kind and result.",
+	}, []string{"kind", "result"})
+	KafkaConsumerLag = prometheus.NewGaugeVec(prometheus.GaugeOpts{
+		Name: "im_kafka_consumer_lag",
+		Help: "Kafka reader lag reported by kafka-go by pod, topic, and consumer group.",
+	}, []string{"pod", "topic", "group"})
+	KafkaBacklogOldestAge = prometheus.NewGaugeVec(prometheus.GaugeOpts{
+		Name: "im_kafka_backlog_oldest_age_seconds",
+		Help: "Age of the latest fetched Kafka message by pod, topic, and consumer group.",
+	}, []string{"pod", "topic", "group"})
+	KafkaDeadLettersTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
+		Name: "im_kafka_dead_letters_total",
+		Help: "Kafka messages moved to dead letter by message kind and reason.",
+	}, []string{"kind", "reason"})
 )
 
 func init() {
-	Registry.MustRegister(WSConnectionsActive, HTTPRequestsTotal, HTTPRequestDuration, DBQueryDuration,
-		RealtimeFanoutEventsTotal, RealtimeFanoutDeliveryDuration)
+	Registry.MustRegister(WSConnectionsActive, IMWSConnectionsActive, IMClusterLivePeers, IMClusterExpectedPeers,
+		HTTPRequestsTotal, HTTPRequestDuration, DBQueryDuration,
+		RealtimeFanoutEventsTotal, RealtimeFanoutDeliveryDuration, ClusterP2MQueueLength,
+		ClusterP2MQueueCapacity, ClusterP2MEnqueueTotal, ClusterRPCInflight, ClusterRPCDuration,
+		ClusterRouteTotal, ClusterProxyResponseTotal, MessagePersistDuration, KafkaMessagePersistTotal)
+	Registry.MustRegister(KafkaConsumerLag, KafkaBacklogOldestAge, KafkaDeadLettersTotal)
+}
+
+func metricPodName() string {
+	if value := strings.TrimSpace(os.Getenv("HOSTNAME")); value != "" {
+		return value
+	}
+	return "standalone"
 }
 
 func ObserveDB(driverName, operation string, started time.Time, err error) {