feat: expose websocket delivery pressure metrics

server/metrics/metrics.go

@@ -54,6 +54,45 @@ var (
 	}, func() float64 {
 		return float64(expvarIntValue("LiveSessions"))
 	})
+	IMWSWriteTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
+		Name: "im_ws_write_total",
+		Help: "WebSocket frame write and pre-write rejection outcomes by pod, frame type, result, and bounded reason.",
+	}, []string{"pod", "frame", "result", "reason"})
+	IMWSWriteDuration = prometheus.NewHistogramVec(prometheus.HistogramOpts{
+		Name:    "im_ws_write_duration_seconds",
+		Help:    "WebSocket frame write duration by pod, frame type, and result.",
+		Buckets: []float64{0.0005, 0.001, 0.003, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10},
+	}, []string{"pod", "frame", "result"})
+	IMWSWriteBytesTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
+		Name: "im_ws_write_bytes_total",
+		Help: "WebSocket payload bytes submitted to the socket by pod and frame type.",
+	}, []string{"pod", "frame"})
+	IMWSDisconnectTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
+		Name: "im_ws_disconnect_total",
+		Help: "WebSocket read-loop disconnects by pod and bounded reason.",
+	}, []string{"pod", "reason"})
+	IMWSSendQueueCurrent = prometheus.NewGaugeVec(prometheus.GaugeOpts{
+		Name: "im_ws_send_queue_current",
+		Help: "Current number of queued WebSocket outbound channel entries across sessions on a pod.",
+	}, []string{"pod"})
+	IMWSSendQueueCapacity = prometheus.NewGaugeVec(prometheus.GaugeOpts{
+		Name: "im_ws_send_queue_capacity",
+		Help: "Configured outbound channel capacity of one WebSocket session by pod.",
+	}, []string{"pod"})
+	IMWSSendQueueEnqueueTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
+		Name: "im_ws_send_queue_enqueue_total",
+		Help: "WebSocket outbound channel enqueue attempts by pod and result.",
+	}, []string{"pod", "result"})
+	IMWSSendQueueDepth = prometheus.NewHistogramVec(prometheus.HistogramOpts{
+		Name:    "im_ws_send_queue_depth",
+		Help:    "Per-session WebSocket outbound channel depth observed after enqueue attempts.",
+		Buckets: []float64{0, 1, 4, 8, 16, 32, 64, 96, 128, 144, 160},
+	}, []string{"pod", "result"})
+	IMWSSendQueueWaitDuration = prometheus.NewHistogramVec(prometheus.HistogramOpts{
+		Name:    "im_ws_send_queue_wait_duration_seconds",
+		Help:    "Time a WebSocket outbound channel entry waited before dequeue or purge.",
+		Buckets: []float64{0.0005, 0.001, 0.003, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60, 120},
+	}, []string{"pod", "result"})
 	IMClusterLivePeers = prometheus.NewGaugeFunc(prometheus.GaugeOpts{
 		Name:        "im_cluster_live_peers",
 		Help:        "Current connected cluster peers.",
@@ -144,7 +183,10 @@ var (
 )
 
 func init() {
-	Registry.MustRegister(WSConnectionsActive, IMWSConnectionsActive, IMClusterLivePeers, IMClusterExpectedPeers,
+	Registry.MustRegister(WSConnectionsActive, IMWSConnectionsActive, IMWSWriteTotal, IMWSWriteDuration,
+		IMWSWriteBytesTotal, IMWSDisconnectTotal, IMWSSendQueueCurrent, IMWSSendQueueCapacity,
+		IMWSSendQueueEnqueueTotal, IMWSSendQueueDepth, IMWSSendQueueWaitDuration,
+		IMClusterLivePeers, IMClusterExpectedPeers,
 		HTTPRequestsTotal, HTTPRequestDuration, DBQueryDuration,
 		RealtimeFanoutEventsTotal, RealtimeFanoutDeliveryDuration, ClusterP2MQueueLength,
 		ClusterP2MQueueCapacity, ClusterP2MEnqueueTotal, ClusterRPCInflight, ClusterRPCDuration,
@@ -203,6 +245,48 @@ func ObserveInternalQueueEnqueue(queue string, depth, capacity int, accepted boo
 	InternalQueueCapacity.WithLabelValues(pod, queue).Set(float64(capacity))
 }
 
+func ObserveWSWrite(frame, result, reason string, started time.Time, payloadBytes int) {
+	pod := metricPodName()
+	frame = normalizeLabel(frame)
+	result = normalizeLabel(result)
+	reason = normalizeLabel(reason)
+	IMWSWriteTotal.WithLabelValues(pod, frame, result, reason).Inc()
+	IMWSWriteDuration.WithLabelValues(pod, frame, result).Observe(time.Since(started).Seconds())
+	if payloadBytes > 0 {
+		IMWSWriteBytesTotal.WithLabelValues(pod, frame).Add(float64(payloadBytes))
+	}
+}
+
+func IncWSDisconnect(reason string) {
+	IMWSDisconnectTotal.WithLabelValues(metricPodName(), normalizeLabel(reason)).Inc()
+}
+
+func ObserveWSSendQueueEnqueue(depth, capacity int, accepted bool) {
+	result := "ok"
+	if !accepted {
+		result = "full"
+	}
+	pod := metricPodName()
+	IMWSSendQueueEnqueueTotal.WithLabelValues(pod, result).Inc()
+	IMWSSendQueueDepth.WithLabelValues(pod, result).Observe(float64(depth))
+	IMWSSendQueueCapacity.WithLabelValues(pod).Set(float64(capacity))
+	if accepted {
+		IMWSSendQueueCurrent.WithLabelValues(pod).Inc()
+	}
+}
+
+func ObserveWSSendQueueDequeue(queuedAt time.Time, purged bool) {
+	result := "dequeued"
+	if purged {
+		result = "purged"
+	}
+	pod := metricPodName()
+	IMWSSendQueueCurrent.WithLabelValues(pod).Dec()
+	if !queuedAt.IsZero() {
+		IMWSSendQueueWaitDuration.WithLabelValues(pod, result).Observe(time.Since(queuedAt).Seconds())
+	}
+}
+
 func normalizeLabel(value string) string {
 	if value == "" {
 		return "unknown"