fix(server): raise topic publish queue capacity

server/hub.go

@@ -10,6 +10,8 @@
 package main
 
 import (
+	"os"
+	"strconv"
 	"strings"
 	"sync"
 	"time"
@@ -30,6 +32,44 @@ var requestLatencyDistribution = []float64{1, 2, 3, 4, 5, 6, 8, 10, 13, 16, 20,
 var outgoingMessageSizeDistribution = []float64{1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 16384,
 	65536, 262144, 1048576, 4194304, 16777216, 67108864, 268435456, 1073741824, 4294967296}
 
+const (
+	defaultTopicClientQueue    = 4096
+	defaultTopicServerQueue    = 512
+	defaultTopicPublishWait    = 200 * time.Millisecond
+	topicClientQueueEnv        = "TOPIC_CLIENT_QUEUE_BUFFER"
+	topicServerQueueEnv        = "TOPIC_SERVER_QUEUE_BUFFER"
+	topicPublishEnqueueWaitEnv = "TOPIC_PUBLISH_ENQUEUE_WAIT_MS"
+)
+
+var topicQueueConfig = loadTopicQueueConfig()
+
+type topicQueueRuntimeConfig struct {
+	clientBuffer       int
+	serverBuffer       int
+	publishEnqueueWait time.Duration
+}
+
+func loadTopicQueueConfig() topicQueueRuntimeConfig {
+	return topicQueueRuntimeConfig{
+		clientBuffer:       intEnvOrDefault(topicClientQueueEnv, defaultTopicClientQueue),
+		serverBuffer:       intEnvOrDefault(topicServerQueueEnv, defaultTopicServerQueue),
+		publishEnqueueWait: time.Duration(intEnvOrDefault(topicPublishEnqueueWaitEnv, int(defaultTopicPublishWait/time.Millisecond))) * time.Millisecond,
+	}
+}
+
+func intEnvOrDefault(name string, fallback int) int {
+	raw := strings.TrimSpace(os.Getenv(name))
+	if raw == "" {
+		return fallback
+	}
+	parsed, err := strconv.Atoi(raw)
+	if err != nil || parsed <= 0 {
+		logs.Warn.Printf("invalid %s=%q, using %d", name, raw, fallback)
+		return fallback
+	}
+	return parsed
+}
+
 // Request to hub to remove the topic
 type topicUnreg struct {
 	// Tenant which owns the topic.
@@ -261,8 +301,8 @@ func (h *Hub) run() {
 					// Indicates a proxy topic.
 					isProxy:   globals.cluster.isRemoteTopic(key),
 					sessions:  make(map[*Session]perSessionData),
-					clientMsg: make(chan *ClientComMessage, 192),
-					serverMsg: make(chan *ServerComMessage, 64),
+					clientMsg: make(chan *ClientComMessage, topicQueueConfig.clientBuffer),
+					serverMsg: make(chan *ServerComMessage, topicQueueConfig.serverBuffer),
 					reg:       make(chan *ClientComMessage, 256),
 					unreg:     make(chan *ClientComMessage, 256),
 					meta:      make(chan *ClientComMessage, 64),

server/session.go

@@ -1019,14 +1019,33 @@ func (s *Session) publish(msg *ClientComMessage) {
 	if sub := s.getSub(msg.RcptTo); sub != nil {
 		// This is a post to a subscribed topic. The message is sent to the topic only
 		msg.queuedAt = time.Now()
-		select {
-		case sub.broadcast <- msg:
-			immetrics.ObserveInternalQueueEnqueue("session_topic_publish", len(sub.broadcast), cap(sub.broadcast), true)
-		default:
-			immetrics.ObserveInternalQueueEnqueue("session_topic_publish", len(sub.broadcast), cap(sub.broadcast), false)
-			// Reply with a 503 to the user.
-			s.queueOut(ErrServiceUnavailableReply(msg, msg.Timestamp))
-			logs.Err.Println("s.publish: sub.broadcast channel full, topic ", msg.RcptTo, s.sid)
+		if topicQueueConfig.publishEnqueueWait > 0 {
+			timer := time.NewTimer(topicQueueConfig.publishEnqueueWait)
+			select {
+			case sub.broadcast <- msg:
+				immetrics.ObserveInternalQueueEnqueue("session_topic_publish", len(sub.broadcast), cap(sub.broadcast), true)
+				if !timer.Stop() {
+					select {
+					case <-timer.C:
+					default:
+					}
+				}
+			case <-timer.C:
+				immetrics.ObserveInternalQueueEnqueue("session_topic_publish", len(sub.broadcast), cap(sub.broadcast), false)
+				// Reply with a 503 to the user.
+				s.queueOut(ErrServiceUnavailableReply(msg, msg.Timestamp))
+				logs.Err.Println("s.publish: sub.broadcast channel full, topic ", msg.RcptTo, s.sid)
+			}
+		} else {
+			select {
+			case sub.broadcast <- msg:
+				immetrics.ObserveInternalQueueEnqueue("session_topic_publish", len(sub.broadcast), cap(sub.broadcast), true)
+			default:
+				immetrics.ObserveInternalQueueEnqueue("session_topic_publish", len(sub.broadcast), cap(sub.broadcast), false)
+				// Reply with a 503 to the user.
+				s.queueOut(ErrServiceUnavailableReply(msg, msg.Timestamp))
+				logs.Err.Println("s.publish: sub.broadcast channel full, topic ", msg.RcptTo, s.sid)
+			}
 		}
 	} else if msg.RcptTo == "sys" {
 		if s.authLvl != auth.LevelRoot {