fix(server): shed overloaded group websocket fanout

server/session.go

@@ -35,6 +35,11 @@ import (
 // Maximum number of queued messages before session is considered stale and dropped.
 const sendQueueLimit = 128
 
+const (
+	lossyBroadcastQueueLimit = sendQueueLimit / 2
+	lossyBroadcastMaxWait    = 2 * time.Second
+)
+
 // Multiplex sessions may receive a short burst of presence updates while many
 // remote connections are closing. Give the cluster drain a bounded chance to
 // free one slot before treating the proxy as stuck.
@@ -547,6 +552,38 @@ func (s *Session) queueOutSharedBytes(data []byte) bool {
 	return true
 }
 
+func (s *Session) queueOutLossySharedBytes(data []byte) (queued bool, skipped bool) {
+	if s == nil || atomic.LoadInt32(&s.terminating) > 0 {
+		return true, false
+	}
+	if s.proto != WEBSOCK {
+		return s.queueOutSharedBytes(data), false
+	}
+
+	s.wsQueueMu.Lock()
+	depth := len(s.send)
+	stale := false
+	if s.wsQueueHead < len(s.wsQueueTimes) {
+		stale = time.Since(s.wsQueueTimes[s.wsQueueHead]) > lossyBroadcastMaxWait
+	}
+	if depth >= lossyBroadcastQueueLimit || stale {
+		immetrics.ObserveWSSendQueueEnqueue(depth, cap(s.send), false)
+		s.wsQueueMu.Unlock()
+		return true, true
+	}
+	select {
+	case s.send <- sharedServerMessage(data):
+		s.wsQueueTimes = append(s.wsQueueTimes, time.Now())
+		immetrics.ObserveWSSendQueueEnqueue(len(s.send), cap(s.send), true)
+		s.wsQueueMu.Unlock()
+		return true, false
+	default:
+		immetrics.ObserveWSSendQueueEnqueue(len(s.send), cap(s.send), false)
+		s.wsQueueMu.Unlock()
+		return true, true
+	}
+}
+
 func (s *Session) maybeScheduleClusterWriteLoop() {
 	if s.multi != nil {
 		s.multi.scheduleClusterWriteLoop()