fix: move group push work off topic loop

server/push.go

@@ -8,14 +8,109 @@
 package main
 
 import (
+	"sync"
+	"sync/atomic"
 	"time"
 
 	"github.com/tinode/chat/server/logs"
+	immetrics "github.com/tinode/chat/server/metrics"
 	"github.com/tinode/chat/server/push"
 	"github.com/tinode/chat/server/store"
 	"github.com/tinode/chat/server/store/types"
 )
 
+const (
+	pushDispatchQueueCapacity = 4096
+	pushDispatchWorkers       = 4
+)
+
+type pushDispatchJob struct {
+	rcpt            *push.Receipt
+	resolvePresence bool
+	queuedAt        time.Time
+}
+
+var asyncPushDispatcher struct {
+	once   sync.Once
+	queues [pushDispatchWorkers]chan pushDispatchJob
+	depth  atomic.Int64
+}
+
+func initAsyncPushDispatcher() {
+	asyncPushDispatcher.once.Do(func() {
+		for i := range pushDispatchWorkers {
+			asyncPushDispatcher.queues[i] = make(chan pushDispatchJob, pushDispatchQueueCapacity/pushDispatchWorkers)
+			go func(queue <-chan pushDispatchJob) {
+				for job := range queue {
+					depth := asyncPushDispatcher.depth.Add(-1)
+					immetrics.SetPushDispatchQueueDepth(int(depth))
+					processPushDispatch(job)
+				}
+			}(asyncPushDispatcher.queues[i])
+		}
+	})
+}
+
+func pushDispatchShard(rcpt *push.Receipt) int {
+	hash := uint32(2166136261)
+	for _, value := range []byte(rcpt.Payload.Topic) {
+		hash ^= uint32(value)
+		hash *= 16777619
+	}
+	hash ^= uint32(rcpt.TenantID)
+	hash *= 16777619
+	return int(hash % pushDispatchWorkers)
+}
+
+func dispatchPushAsync(rcpt *push.Receipt, resolvePresence bool) bool {
+	if rcpt == nil || globals.usersUpdate == nil {
+		return false
+	}
+	initAsyncPushDispatcher()
+	queue := asyncPushDispatcher.queues[pushDispatchShard(rcpt)]
+	job := pushDispatchJob{rcpt: rcpt, resolvePresence: resolvePresence, queuedAt: time.Now()}
+	depth := asyncPushDispatcher.depth.Add(1)
+	select {
+	case queue <- job:
+		immetrics.ObservePushDispatchEnqueue(int(asyncPushDispatcher.depth.Load()), true)
+		return true
+	default:
+		depth = asyncPushDispatcher.depth.Add(-1)
+		immetrics.ObservePushDispatchEnqueue(int(depth), false)
+		logs.Err.Printf("push dispatch queue full: tenant=%d topic=%s recipients=%d shard_len=%d shard_cap=%d total_depth=%d",
+			rcpt.TenantID, rcpt.Payload.Topic, len(rcpt.To), len(queue), cap(queue), depth)
+		return false
+	}
+}
+
+func processPushDispatch(job pushDispatchJob) {
+	started := time.Now()
+	var dispatchErr error
+	if job.resolvePresence {
+		unresolved := make([]types.Uid, 0, len(job.rcpt.To))
+		for uid, recipient := range job.rcpt.To {
+			if recipient.Delivered == 0 {
+				unresolved = append(unresolved, uid)
+			}
+		}
+		presences, err := store.GetUsersPresence(job.rcpt.TenantID, unresolved)
+		if err != nil {
+			dispatchErr = err
+			logs.Warn.Printf("push dispatch: failed to read presence topic=%s recipients=%d err=%v",
+				job.rcpt.Payload.Topic, len(unresolved), err)
+		}
+		for uid, presence := range presences {
+			if presence != nil && presence.Online && presence.SessionCount > 0 {
+				recipient := job.rcpt.To[uid]
+				recipient.Delivered = presence.SessionCount
+				job.rcpt.To[uid] = recipient
+			}
+		}
+	}
+	sendPush(job.rcpt)
+	immetrics.ObservePushDispatch(job.queuedAt, len(job.rcpt.To), started, dispatchErr)
+}
+
 // Subscribe or unsubscribe user to/from FCM topic (channel).
 func (t *Topic) channelSubUnsub(uid types.Uid, sub bool) {
 	push.ChannelSub(&push.ChannelReq{
@@ -75,15 +170,6 @@ func (t *Topic) pushForData(fromUid types.Uid, data *MsgServerData, msgMarkedAsR
 			// Make sure the sender's devices receive a silent push.
 			online = 1
 		}
-		if online == 0 {
-			presence, err := store.GetUserPresence(t.tenantID, uid)
-			if err != nil {
-				logs.Warn.Printf("topic[%s]: failed to read user presence for push: uid=%s err=%v", t.name, uid.UserId(), err)
-			} else if presence != nil && presence.Online && presence.SessionCount > 0 {
-				online = presence.SessionCount
-			}
-		}
-
 		// Send only to those who have notifications enabled.
 		mode := pud.modeWant & pud.modeGiven
 		if mode.IsPresencer() && mode.IsReader() && !pud.deleted && !pud.isChan {