fix: move group push work off topic loop

server/hub.go

@@ -57,6 +57,12 @@ type userStatusReq struct {
 	state types.ObjState
 }
 
+type deferredPresenceKey struct {
+	tenantID types.TenantID
+	topic    string
+	source   string
+}
+
 // Hub is the core structure which holds topics.
 type Hub struct {
 
@@ -74,6 +80,9 @@ type Hub struct {
 
 	// Channel for routing server-generated messages, buffered at 4096
 	routeSrv chan *ServerComMessage
+	// Presence updates which could not immediately enter a topic queue.
+	deferredPresence     chan *ServerComMessage
+	deferredPresenceStop chan struct{}
 
 	// subscribe session to topic, possibly creating a new topic, buffered at 256
 	join chan *ClientComMessage
@@ -121,14 +130,16 @@ func newHub() *Hub {
 	h := &Hub{
 		topics: &sync.Map{},
 		// TODO: verify if these channels have to be buffered.
-		routeCli:   make(chan *ClientComMessage, 4096),
-		routeSrv:   make(chan *ServerComMessage, 4096),
-		join:       make(chan *ClientComMessage, 256),
-		unreg:      make(chan *topicUnreg, 256),
-		rehash:     make(chan bool),
-		meta:       make(chan *ClientComMessage, 128),
-		userStatus: make(chan *userStatusReq, 128),
-		shutdown:   make(chan chan<- bool),
+		routeCli:             make(chan *ClientComMessage, 4096),
+		routeSrv:             make(chan *ServerComMessage, 4096),
+		deferredPresence:     make(chan *ServerComMessage, 8192),
+		deferredPresenceStop: make(chan struct{}),
+		join:                 make(chan *ClientComMessage, 256),
+		unreg:                make(chan *topicUnreg, 256),
+		rehash:               make(chan bool),
+		meta:                 make(chan *ClientComMessage, 128),
+		userStatus:           make(chan *userStatusReq, 128),
+		shutdown:             make(chan chan<- bool),
 	}
 
 	statsRegisterInt("LiveTopics")
@@ -154,11 +165,70 @@ func newHub() *Hub {
 	statsRegisterHistogram("RequestLatency", requestLatencyDistribution)
 	statsRegisterHistogram("OutgoingMessageSize", outgoingMessageSizeDistribution)
 
+	go h.runDeferredPresence()
 	go h.run()
 
 	return h
 }
 
+func (h *Hub) deferPresence(msg *ServerComMessage) bool {
+	select {
+	case h.deferredPresence <- msg:
+		immetrics.ObserveInternalQueueEnqueue("topic_presence_deferred", len(h.deferredPresence), cap(h.deferredPresence), true)
+		return true
+	default:
+		immetrics.ObserveInternalQueueEnqueue("topic_presence_deferred", len(h.deferredPresence), cap(h.deferredPresence), false)
+		return false
+	}
+}
+
+func (h *Hub) runDeferredPresence() {
+	const maxPending = 16384
+	ticker := time.NewTicker(5 * time.Millisecond)
+	defer ticker.Stop()
+	pending := make(map[deferredPresenceKey]*ServerComMessage)
+
+	for {
+		select {
+		case msg := <-h.deferredPresence:
+			if msg == nil || msg.Pres == nil {
+				continue
+			}
+			key := deferredPresenceKey{tenantID: msg.TenantID, topic: msg.RcptTo, source: msg.Pres.Src}
+			if _, exists := pending[key]; exists || len(pending) < maxPending {
+				pending[key] = msg
+			} else {
+				immetrics.ObserveInternalQueueEnqueue("topic_presence_pending", len(pending), maxPending, false)
+				logs.Err.Printf("hub: deferred presence queue full topic=%s tenant=%d source=%s",
+					msg.RcptTo, msg.TenantID, msg.Pres.Src)
+			}
+			immetrics.SetInternalQueueCurrent("topic_presence_pending", len(pending))
+		case <-ticker.C:
+			budget := 64
+			for key, msg := range pending {
+				dst := h.topicGet(types.TopicKey{TenantID: key.tenantID, Topic: key.topic})
+				if dst == nil || dst.serverMsg == nil {
+					delete(pending, key)
+					continue
+				}
+				select {
+				case dst.serverMsg <- msg:
+					immetrics.ObserveInternalQueueEnqueue("topic_server", len(dst.serverMsg), cap(dst.serverMsg), true)
+					delete(pending, key)
+					budget--
+				default:
+				}
+				if budget == 0 {
+					break
+				}
+			}
+			immetrics.SetInternalQueueCurrent("topic_presence_pending", len(pending))
+		case <-h.deferredPresenceStop:
+			return
+		}
+	}
+}
+
 func (h *Hub) run() {
 	for {
 		select {
@@ -246,6 +316,7 @@ func (h *Hub) run() {
 			if dst := h.topicGet(key); dst != nil {
 				// Everything is OK, sending packet to known topic
 				if dst.clientMsg != nil {
+					msg.queuedAt = time.Now()
 					select {
 					case dst.clientMsg <- msg:
 						immetrics.ObserveInternalQueueEnqueue("topic_client", len(dst.clientMsg), cap(dst.clientMsg), true)
@@ -284,8 +355,10 @@ func (h *Hub) run() {
 				case dst.serverMsg <- msg:
 					immetrics.ObserveInternalQueueEnqueue("topic_server", len(dst.serverMsg), cap(dst.serverMsg), true)
 				default:
-					immetrics.ObserveInternalQueueEnqueue("topic_server", len(dst.serverMsg), cap(dst.serverMsg), false)
-					logs.Err.Println("hub: topic's broadcast queue is full", dst.name)
+					if msg.Pres == nil || !h.deferPresence(msg) {
+						immetrics.ObserveInternalQueueEnqueue("topic_server", len(dst.serverMsg), cap(dst.serverMsg), false)
+						logs.Err.Println("hub: topic's broadcast queue is full", dst.name)
+					}
 				}
 			} else if (strings.HasPrefix(msg.RcptTo, "usr") || strings.HasPrefix(msg.RcptTo, "grp")) &&
 				globals.cluster.isRemoteTopic(key) {
@@ -357,6 +430,9 @@ func (h *Hub) run() {
 			}
 
 			logs.Info.Printf("Hub shutdown completed with %d topics", topicCount)
+			if h.deferredPresenceStop != nil {
+				close(h.deferredPresenceStop)
+			}
 
 			// let the main goroutine know we are done with the cleanup
 			hubdone <- true