fix: move group push work off topic loop

server/metrics/metrics.go

@@ -154,6 +154,10 @@ var (
 		Name: "im_internal_queue_capacity",
 		Help: "Configured internal message queue capacity by pod and queue.",
 	}, []string{"pod", "queue"})
+	InternalQueueCurrent = prometheus.NewGaugeVec(prometheus.GaugeOpts{
+		Name: "im_internal_queue_current",
+		Help: "Current entries in an internal deferred queue by pod and queue.",
+	}, []string{"pod", "queue"})
 	MessagePersistDuration = prometheus.NewHistogramVec(prometheus.HistogramOpts{
 		Name:    "im_message_persist_duration_seconds",
 		Help:    "Kafka message persistence duration by message kind and result.",
@@ -164,6 +168,34 @@ var (
 		Help:    "Message publish hot-path duration by message kind, stage, and result.",
 		Buckets: []float64{0.001, 0.003, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10},
 	}, []string{"kind", "stage", "result"})
+	TopicQueueWaitDuration = prometheus.NewHistogramVec(prometheus.HistogramOpts{
+		Name:    "im_topic_queue_wait_duration_seconds",
+		Help:    "Time a client publish waited in a topic queue before processing by topic kind.",
+		Buckets: []float64{0.0005, 0.001, 0.003, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60},
+	}, []string{"pod", "kind"})
+	PushDispatchQueueEnqueueTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
+		Name: "im_push_dispatch_queue_enqueue_total",
+		Help: "Asynchronous push dispatch queue enqueue attempts by pod and result.",
+	}, []string{"pod", "result"})
+	PushDispatchQueueDepth = prometheus.NewGaugeVec(prometheus.GaugeOpts{
+		Name: "im_push_dispatch_queue_depth",
+		Help: "Current asynchronous push dispatch queue depth by pod.",
+	}, []string{"pod"})
+	PushDispatchQueueWaitDuration = prometheus.NewHistogramVec(prometheus.HistogramOpts{
+		Name:    "im_push_dispatch_queue_wait_duration_seconds",
+		Help:    "Time a push dispatch job waited before a worker started processing it.",
+		Buckets: []float64{0.0005, 0.001, 0.003, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60},
+	}, []string{"pod"})
+	PushDispatchDuration = prometheus.NewHistogramVec(prometheus.HistogramOpts{
+		Name:    "im_push_dispatch_duration_seconds",
+		Help:    "Asynchronous push presence resolution and dispatch duration by pod and result.",
+		Buckets: []float64{0.001, 0.003, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60},
+	}, []string{"pod", "result"})
+	PushRecipientCount = prometheus.NewHistogramVec(prometheus.HistogramOpts{
+		Name:    "im_push_recipient_count",
+		Help:    "Recipients in an asynchronously dispatched push receipt by pod.",
+		Buckets: []float64{1, 2, 10, 50, 100, 300, 600, 900, 1200, 2000},
+	}, []string{"pod"})
 	KafkaMessagePersistTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
 		Name: "im_kafka_message_persist_total",
 		Help: "Kafka message persistence outcomes by message kind and result.",
@@ -191,7 +223,9 @@ func init() {
 		RealtimeFanoutEventsTotal, RealtimeFanoutDeliveryDuration, ClusterP2MQueueLength,
 		ClusterP2MQueueCapacity, ClusterP2MEnqueueTotal, ClusterRPCInflight, ClusterRPCDuration,
 		ClusterRouteTotal, ClusterProxyResponseTotal, InternalQueueEnqueueTotal, InternalQueueDepth,
-		InternalQueueCapacity, MessagePersistDuration, MessagePublishStageDuration,
+		InternalQueueCapacity, InternalQueueCurrent, MessagePersistDuration, MessagePublishStageDuration, TopicQueueWaitDuration,
+		PushDispatchQueueEnqueueTotal, PushDispatchQueueDepth, PushDispatchQueueWaitDuration,
+		PushDispatchDuration, PushRecipientCount,
 		KafkaMessagePersistTotal)
 	Registry.MustRegister(KafkaConsumerLag, KafkaBacklogOldestAge, KafkaDeadLettersTotal)
 }
@@ -220,6 +254,39 @@ func ObserveMessagePublishStage(kind, stage string, started time.Time, err error
 		Observe(time.Since(started).Seconds())
 }
 
+func ObserveTopicQueueWait(kind string, queuedAt time.Time) {
+	if queuedAt.IsZero() {
+		return
+	}
+	TopicQueueWaitDuration.WithLabelValues(metricPodName(), normalizeLabel(kind)).Observe(time.Since(queuedAt).Seconds())
+}
+
+func ObservePushDispatchEnqueue(depth int, accepted bool) {
+	result := "ok"
+	if !accepted {
+		result = "full"
+	}
+	PushDispatchQueueEnqueueTotal.WithLabelValues(metricPodName(), result).Inc()
+	PushDispatchQueueDepth.WithLabelValues(metricPodName()).Set(float64(depth))
+}
+
+func SetPushDispatchQueueDepth(depth int) {
+	PushDispatchQueueDepth.WithLabelValues(metricPodName()).Set(float64(depth))
+}
+
+func ObservePushDispatch(queuedAt time.Time, recipients int, started time.Time, err error) {
+	pod := metricPodName()
+	if !queuedAt.IsZero() {
+		PushDispatchQueueWaitDuration.WithLabelValues(pod).Observe(started.Sub(queuedAt).Seconds())
+	}
+	result := "ok"
+	if err != nil {
+		result = "error"
+	}
+	PushDispatchDuration.WithLabelValues(pod, result).Observe(time.Since(started).Seconds())
+	PushRecipientCount.WithLabelValues(pod).Observe(float64(recipients))
+}
+
 func IncRealtimeFanout(stage, kind, targetType, status string) {
 	RealtimeFanoutEventsTotal.WithLabelValues(normalizeLabel(stage), normalizeLabel(kind),
 		normalizeLabel(targetType), normalizeLabel(status)).Inc()
@@ -245,6 +312,10 @@ func ObserveInternalQueueEnqueue(queue string, depth, capacity int, accepted boo
 	InternalQueueCapacity.WithLabelValues(pod, queue).Set(float64(capacity))
 }
 
+func SetInternalQueueCurrent(queue string, depth int) {
+	InternalQueueCurrent.WithLabelValues(metricPodName(), normalizeLabel(queue)).Set(float64(depth))
+}
+
 func ObserveWSWrite(frame, result, reason string, started time.Time, payloadBytes int) {
 	pod := metricPodName()
 	frame = normalizeLabel(frame)