perf(group): reduce websocket fanout serialization

server/metrics/metrics.go

@@ -190,6 +190,45 @@ var (
 		Help:    "Time a client publish waited in a topic queue before processing by topic kind.",
 		Buckets: []float64{0.0005, 0.001, 0.003, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60},
 	}, []string{"pod", "kind"})
+	TopicProcessDuration = prometheus.NewHistogramVec(prometheus.HistogramOpts{
+		Name:    "im_topic_process_duration_seconds",
+		Help:    "End-to-end topic event processing duration by pod, topic kind, role, and operation.",
+		Buckets: []float64{0.0005, 0.001, 0.003, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60},
+	}, []string{"pod", "kind", "role", "operation"})
+	TopicBroadcastDuration = prometheus.NewHistogramVec(prometheus.HistogramOpts{
+		Name:    "im_topic_broadcast_duration_seconds",
+		Help:    "Time spent preparing and enqueueing one topic broadcast by pod, topic kind, and role.",
+		Buckets: []float64{0.0005, 0.001, 0.003, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10},
+	}, []string{"pod", "kind", "role"})
+	TopicBroadcastTargets = prometheus.NewHistogramVec(prometheus.HistogramOpts{
+		Name:    "im_topic_broadcast_targets",
+		Help:    "Number of local WebSocket, multiplex, or other targets considered by one topic broadcast.",
+		Buckets: []float64{0, 1, 10, 50, 100, 250, 500, 750, 1000, 1500, 2000, 3000},
+	}, []string{"pod", "kind", "role", "target_type"})
+	TopicSharedSerializationTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
+		Name: "im_topic_shared_serialization_total",
+		Help: "Shared WebSocket serialization events by pod, topic kind, and result.",
+	}, []string{"pod", "kind", "result"})
+	ClusterM2PQueueWaitDuration = prometheus.NewHistogramVec(prometheus.HistogramOpts{
+		Name:    "im_cluster_m2p_queue_wait_duration_seconds",
+		Help:    "Time a master-to-proxy response waited in the multiplex outbound queue by pod and peer.",
+		Buckets: []float64{0.0005, 0.001, 0.003, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10},
+	}, []string{"pod", "peer"})
+	ClusterM2PQueueDepth = prometheus.NewHistogramVec(prometheus.HistogramOpts{
+		Name:    "im_cluster_m2p_queue_depth",
+		Help:    "Master-to-proxy multiplex outbound queue depth observed after enqueue by pod and peer.",
+		Buckets: []float64{0, 1, 4, 8, 16, 32, 64, 96, 128, 160},
+	}, []string{"pod", "peer"})
+	ClusterProxyQueueWaitDuration = prometheus.NewHistogramVec(prometheus.HistogramOpts{
+		Name:    "im_cluster_proxy_queue_wait_duration_seconds",
+		Help:    "Time a cluster response waited in the proxy topic queue by pod and message type.",
+		Buckets: []float64{0.0005, 0.001, 0.003, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60},
+	}, []string{"pod", "message_type"})
+	WSReadDispatchDuration = prometheus.NewHistogramVec(prometheus.HistogramOpts{
+		Name:    "im_ws_read_dispatch_duration_seconds",
+		Help:    "Time from a completed WebSocket frame read through JSON decoding and local dispatch.",
+		Buckets: []float64{0.0001, 0.0005, 0.001, 0.003, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5},
+	}, []string{"pod"})
 	PushDispatchQueueEnqueueTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
 		Name: "im_push_dispatch_queue_enqueue_total",
 		Help: "Asynchronous push dispatch queue enqueue attempts by pod and result.",
@@ -242,6 +281,8 @@ func init() {
 		ClusterP2MQueueCapacity, ClusterP2MEnqueueTotal, ClusterRPCInflight, ClusterRPCDuration,
 		ClusterRouteTotal, ClusterProxyResponseTotal, InternalQueueEnqueueTotal, InternalQueueDepth,
 		InternalQueueCapacity, InternalQueueCurrent, MessagePersistDuration, MessagePublishStageDuration, TopicQueueWaitDuration,
+		TopicProcessDuration, TopicBroadcastDuration, TopicBroadcastTargets, TopicSharedSerializationTotal,
+		ClusterM2PQueueWaitDuration, ClusterM2PQueueDepth, ClusterProxyQueueWaitDuration, WSReadDispatchDuration,
 		PushDispatchQueueEnqueueTotal, PushDispatchQueueDepth, PushDispatchQueueWaitDuration,
 		PushDispatchDuration, PushRecipientCount,
 		KafkaMessagePersistTotal)
@@ -279,6 +320,47 @@ func ObserveTopicQueueWait(kind string, queuedAt time.Time) {
 	TopicQueueWaitDuration.WithLabelValues(metricPodName(), normalizeLabel(kind)).Observe(time.Since(queuedAt).Seconds())
 }
 
+func ObserveTopicProcess(kind, role, operation string, started time.Time) {
+	TopicProcessDuration.WithLabelValues(metricPodName(), normalizeLabel(kind), normalizeLabel(role),
+		normalizeLabel(operation)).Observe(time.Since(started).Seconds())
+}
+
+func ObserveTopicBroadcast(kind, role string, started time.Time, websocketTargets, multiplexTargets, otherTargets int) {
+	pod := metricPodName()
+	kind = normalizeLabel(kind)
+	role = normalizeLabel(role)
+	TopicBroadcastDuration.WithLabelValues(pod, kind, role).Observe(time.Since(started).Seconds())
+	TopicBroadcastTargets.WithLabelValues(pod, kind, role, "websocket").Observe(float64(websocketTargets))
+	TopicBroadcastTargets.WithLabelValues(pod, kind, role, "multiplex").Observe(float64(multiplexTargets))
+	TopicBroadcastTargets.WithLabelValues(pod, kind, role, "other").Observe(float64(otherTargets))
+}
+
+func IncTopicSharedSerialization(kind, result string) {
+	TopicSharedSerializationTotal.WithLabelValues(metricPodName(), normalizeLabel(kind), normalizeLabel(result)).Inc()
+}
+
+func ObserveClusterM2PEnqueue(peer string, depth int) {
+	ClusterM2PQueueDepth.WithLabelValues(metricPodName(), normalizeLabel(peer)).Observe(float64(depth))
+}
+
+func ObserveClusterM2PDequeue(peer string, queuedAt time.Time) {
+	if queuedAt.IsZero() {
+		return
+	}
+	ClusterM2PQueueWaitDuration.WithLabelValues(metricPodName(), normalizeLabel(peer)).Observe(time.Since(queuedAt).Seconds())
+}
+
+func ObserveClusterProxyQueueWait(messageType string, queuedAt time.Time) {
+	if queuedAt.IsZero() {
+		return
+	}
+	ClusterProxyQueueWaitDuration.WithLabelValues(metricPodName(), normalizeLabel(messageType)).Observe(time.Since(queuedAt).Seconds())
+}
+
+func ObserveWSReadDispatch(started time.Time) {
+	WSReadDispatchDuration.WithLabelValues(metricPodName()).Observe(time.Since(started).Seconds())
+}
+
 func ObservePushDispatchEnqueue(depth int, accepted bool) {
 	result := "ok"
 	if !accepted {