fix: harden websocket heartbeat under load

This commit is contained in:
richjohnrambo
2026-09-30 17:36:33 +08:00
parent b83800c0f3
commit 2b631ab176
12 changed files with 228 additions and 9 deletions
+20 -1
View File
@@ -71,6 +71,14 @@ var (
Name: "im_ws_disconnect_total",
Help: "WebSocket read-loop disconnects by pod and bounded reason.",
}, []string{"pod", "reason"})
IMWSHeartbeatTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
Name: "im_ws_heartbeat_total",
Help: "WebSocket heartbeat events by pod and event.",
}, []string{"pod", "event"})
IMWSHeartbeatSeconds = prometheus.NewGaugeVec(prometheus.GaugeOpts{
Name: "im_ws_heartbeat_seconds",
Help: "Effective WebSocket heartbeat timing by pod and setting.",
}, []string{"pod", "setting"})
IMWSSendQueueCurrent = prometheus.NewGaugeVec(prometheus.GaugeOpts{
Name: "im_ws_send_queue_current",
Help: "Current number of queued WebSocket outbound channel entries across sessions on a pod.",
@@ -216,7 +224,8 @@ var (
func init() {
Registry.MustRegister(WSConnectionsActive, IMWSConnectionsActive, IMWSWriteTotal, IMWSWriteDuration,
IMWSWriteBytesTotal, IMWSDisconnectTotal, IMWSSendQueueCurrent, IMWSSendQueueCapacity,
IMWSWriteBytesTotal, IMWSDisconnectTotal, IMWSHeartbeatTotal, IMWSHeartbeatSeconds,
IMWSSendQueueCurrent, IMWSSendQueueCapacity,
IMWSSendQueueEnqueueTotal, IMWSSendQueueDepth, IMWSSendQueueWaitDuration,
IMClusterLivePeers, IMClusterExpectedPeers,
HTTPRequestsTotal, HTTPRequestDuration, DBQueryDuration,
@@ -332,6 +341,16 @@ func IncWSDisconnect(reason string) {
IMWSDisconnectTotal.WithLabelValues(metricPodName(), normalizeLabel(reason)).Inc()
}
func IncWSHeartbeat(event string) {
IMWSHeartbeatTotal.WithLabelValues(metricPodName(), normalizeLabel(event)).Inc()
}
func SetWSHeartbeatConfig(pingPeriod, pongWait time.Duration) {
pod := metricPodName()
IMWSHeartbeatSeconds.WithLabelValues(pod, "ping_period").Set(pingPeriod.Seconds())
IMWSHeartbeatSeconds.WithLabelValues(pod, "pong_wait").Set(pongWait.Seconds())
}
func ObserveWSSendQueueEnqueue(depth, capacity int, accepted bool) {
result := "ok"
if !accepted {