fix: trace delayed websocket acknowledgements

server/cluster.go

@@ -1215,6 +1215,7 @@ func clusterInit(configString json.RawMessage, self *string) int {
 
 func registerClusterGobTypes() {
 	gob.Register([]any{})
+	gob.Register([]map[string]any{})
 	gob.Register(map[string]any{})
 	gob.Register(map[string]int{})
 	gob.Register(map[string]string{})

server/cluster_gob_test.go

@@ -28,6 +28,7 @@ func TestClusterGobRegistersSystemMessagePayloadTypes(t *testing.T) {
 				Content: map[string]any{
 					"text":     "成员加入了群聊",
 					"template": "group_member_joined",
+					"items":    []map[string]any{{"type": "member", "id": "usr123"}},
 					"actors": []imSystemMessageActor{{
 						UserID: "usr123",
 						Click:  map[string]any{"type": "user_profile", "target_id": "usr123"},

server/hdl_websock.go

@@ -148,12 +148,18 @@ func (sess *Session) writeLoop() {
 					if !sess.sendMessage(w) {
 						return
 					}
+					if isAcceptedCtrl(msg) {
+						immetrics.ObserveCtrlACKDelivery("write", msg.Timestamp)
+					}
 				}
 			case *ServerComMessage: // single unserialized message
 				w := sess.serializeAndUpdateStats(v)
 				if !sess.sendMessage(w) {
 					return
 				}
+				if isAcceptedCtrl(v) {
+					immetrics.ObserveCtrlACKDelivery("write", v.Timestamp)
+				}
 			default: // serialized message
 				if !sess.sendMessage(v) {
 					return

server/metrics/metrics.go

@@ -105,6 +105,11 @@ var (
 		Help:    "Time a WebSocket outbound channel entry waited before dequeue or purge.",
 		Buckets: []float64{0.0005, 0.001, 0.003, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60, 120},
 	}, []string{"pod", "result"})
+	IMCtrlACKDeliveryDuration = prometheus.NewHistogramVec(prometheus.HistogramOpts{
+		Name:    "im_ctrl_ack_delivery_duration_seconds",
+		Help:    "Accepted control-response age from request receipt to WebSocket enqueue or completed socket write.",
+		Buckets: []float64{0.0005, 0.001, 0.003, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 20, 30, 60, 120},
+	}, []string{"pod", "stage"})
 	IMClusterLivePeers = prometheus.NewGaugeFunc(prometheus.GaugeOpts{
 		Name:        "im_cluster_live_peers",
 		Help:        "Current connected cluster peers.",
@@ -230,7 +235,7 @@ func init() {
 	Registry.MustRegister(WSConnectionsActive, IMWSConnectionsActive, IMWSWriteTotal, IMWSWriteDuration,
 		IMWSWriteBytesTotal, IMWSDisconnectTotal, IMWSHeartbeatTotal, IMWSHeartbeatSeconds,
 		IMWSSendQueueCurrent, IMWSSendQueueCapacity,
-		IMWSSendQueueEnqueueTotal, IMWSSendQueueDepth, IMWSSendQueueWaitDuration,
+		IMWSSendQueueEnqueueTotal, IMWSSendQueueDepth, IMWSSendQueueWaitDuration, IMCtrlACKDeliveryDuration,
 		IMClusterLivePeers, IMClusterExpectedPeers,
 		HTTPRequestsTotal, HTTPRequestDuration, DBQueryDuration, DBInvalidConnectionRetryTotal,
 		RealtimeFanoutEventsTotal, RealtimeFanoutDeliveryDuration, ClusterP2MQueueLength,
@@ -385,6 +390,13 @@ func ObserveWSSendQueueDequeue(queuedAt time.Time, purged bool) {
 	}
 }
 
+func ObserveCtrlACKDelivery(stage string, requestAt time.Time) {
+	if requestAt.IsZero() {
+		return
+	}
+	IMCtrlACKDeliveryDuration.WithLabelValues(metricPodName(), normalizeLabel(stage)).Observe(time.Since(requestAt).Seconds())
+}
+
 func normalizeLabel(value string) string {
 	if value == "" {
 		return "unknown"

server/monitoring_http_test.go

@@ -96,6 +96,7 @@ func TestServeMetricsIncludesWSConnections(t *testing.T) {
 	immetrics.IncWSDisconnect("close_normal")
 	immetrics.ObserveWSSendQueueEnqueue(1, 160, true)
 	immetrics.ObserveWSSendQueueDequeue(time.Now(), false)
+	immetrics.ObserveCtrlACKDelivery("write", time.Now())
 	immetrics.ObserveTopicQueueWait("group", time.Now())
 	immetrics.ObservePushDispatchEnqueue(1, true)
 	immetrics.ObservePushDispatch(time.Now(), 1, time.Now(), nil)
@@ -113,6 +114,7 @@ func TestServeMetricsIncludesWSConnections(t *testing.T) {
 		"im_ws_write_total", "im_ws_write_duration_seconds", "im_ws_write_bytes_total",
 		"im_ws_disconnect_total", "im_ws_send_queue_current", "im_ws_send_queue_capacity",
 		"im_ws_send_queue_enqueue_total", "im_ws_send_queue_depth", "im_ws_send_queue_wait_duration_seconds",
+		"im_ctrl_ack_delivery_duration_seconds",
 		"im_cluster_p2m_enqueue_total", "im_cluster_rpc_duration_seconds", "im_cluster_route_total",
 		"im_internal_queue_enqueue_total", "im_internal_queue_depth", "im_internal_queue_capacity", "im_internal_queue_current",
 		"im_topic_queue_wait_duration_seconds", "im_push_dispatch_queue_enqueue_total",

server/session.go

@@ -447,12 +447,19 @@ func (s *Session) queueOut(msg *ServerComMessage) bool {
 		logs.Err.Println("s.queueOut: session's send queue full", s.sid)
 		return false
 	}
+	if isAcceptedCtrl(msg) && s.proto == WEBSOCK {
+		immetrics.ObserveCtrlACKDelivery("enqueue", msg.Timestamp)
+	}
 	if s.isMultiplex() {
 		s.scheduleClusterWriteLoop()
 	}
 	return true
 }
 
+func isAcceptedCtrl(msg *ServerComMessage) bool {
+	return msg != nil && msg.Ctrl != nil && msg.Ctrl.Code == http.StatusAccepted && msg.Ctrl.Id != ""
+}
+
 // queueOutBytes attempts to send a ServerComMessage already serialized to []byte.
 // If the send buffer is full, it fails.
 func (s *Session) queueOutBytes(data []byte) bool {