perf(group): reduce websocket fanout serialization

server/cluster.go

@@ -284,6 +284,9 @@ type ClusterResp struct {
 
 	// Original request type.
 	OrigReqType ProxyReqType
+
+	// Set after RPC decoding on the receiving node; not serialized by gob.
+	proxyQueuedAt time.Time
 }
 
 // ClusterPing is used to detect node restarts.
@@ -743,6 +746,7 @@ func (Cluster) TopicProxy(msg *ClusterResp, unused *bool) error {
 	// Find appropriate topic, send the message to it.
 	if t := globals.hub.topicGet(types.TopicKey{TenantID: msg.TenantID, Topic: msg.RcptTo}); t != nil {
 		msg.SrvMsg.uid = types.ParseUserId(msg.SrvMsg.AsUser)
+		msg.proxyQueuedAt = time.Now()
 		select {
 		case t.proxy <- msg:
 			immetrics.ObserveInternalQueueEnqueue("cluster_topic_proxy", len(t.proxy), cap(t.proxy), true)
@@ -1420,6 +1424,8 @@ func (sess *Session) clusterWriteLoop(forTopic string) {
 				return
 			}
 			srvMsg := msg.(*ServerComMessage)
+			immetrics.ObserveClusterM2PDequeue(sess.clnode.name, srvMsg.clusterQueuedAt)
+			srvMsg.clusterQueuedAt = time.Time{}
 			response := &ClusterResp{TenantID: srvMsg.TenantID, SrvMsg: srvMsg}
 			if srvMsg.sess == nil {
 				response.OrigSid = "*"