fix: harden multi-pod delivery and load verification

server/topic_proxy.go

@@ -9,12 +9,34 @@ package main
 import (
 	"errors"
 	"net/http"
+	"sync/atomic"
 	"time"
 
 	"github.com/tinode/chat/server/logs"
+	immetrics "github.com/tinode/chat/server/metrics"
 	"github.com/tinode/chat/server/store/types"
 )
 
+var (
+	clusterProxyOverloadLogAt      atomic.Int64
+	clusterProxyOverloadSuppressed atomic.Int64
+)
+
+func logClusterProxyRouteFailure(topic, operation string, err error) {
+	if !errors.Is(err, errClusterLoadExceeded) && !errors.Is(err, errClusterOverloaded) {
+		logs.Warn.Printf("topic proxy[%s]: route %s failed - %s", topic, operation, err)
+		return
+	}
+	now := time.Now().Unix()
+	previous := clusterProxyOverloadLogAt.Load()
+	if previous == now || !clusterProxyOverloadLogAt.CompareAndSwap(previous, now) {
+		clusterProxyOverloadSuppressed.Add(1)
+		return
+	}
+	logs.Warn.Printf("topic proxy overload: topic=%s operation=%s suppressed=%d", topic, operation,
+		clusterProxyOverloadSuppressed.Swap(0))
+}
+
 func (t *Topic) runProxy(hub *Hub) {
 	killTimer := time.NewTimer(time.Hour)
 	killTimer.Stop()
@@ -27,8 +49,8 @@ func (t *Topic) runProxy(hub *Hub) {
 				msg.sess.queueOut(ErrLockedReply(msg, types.TimeNow()))
 			} else if err := globals.cluster.routeToTopicMaster(ProxyReqJoin, msg, t.name, msg.sess); err != nil {
 				// Response (ctrl message) will be handled when it's received via the proxy channel.
-				logs.Warn.Printf("proxy topic[%s]: route join request from proxy to master failed - %s", t.name, err)
-				msg.sess.queueOut(ErrClusterUnreachableReply(msg, types.TimeNow()))
+				logClusterProxyRouteFailure(t.name, "join", err)
+				msg.sess.queueOut(clusterRouteErrorReply(msg, err))
 			}
 			if msg.sess.inflightReqs != nil {
 				msg.sess.inflightReqs.Done()
@@ -51,8 +73,8 @@ func (t *Topic) runProxy(hub *Hub) {
 		case msg := <-t.clientMsg:
 			// Content message intended for broadcasting to recipients
 			if err := globals.cluster.routeToTopicMaster(ProxyReqBroadcast, msg, t.name, msg.sess); err != nil {
-				logs.Warn.Printf("topic proxy[%s]: route broadcast request from proxy to master failed - %s", t.name, err)
-				msg.sess.queueOut(ErrClusterUnreachableReply(msg, types.TimeNow()))
+				logClusterProxyRouteFailure(t.name, "broadcast", err)
+				msg.sess.queueOut(clusterRouteErrorReply(msg, err))
 			}
 
 		case msg := <-t.serverMsg:
@@ -71,8 +93,8 @@ func (t *Topic) runProxy(hub *Hub) {
 		case msg := <-t.meta:
 			// Request to get/set topic metadata
 			if err := globals.cluster.routeToTopicMaster(ProxyReqMeta, msg, t.name, msg.sess); err != nil {
-				logs.Warn.Printf("proxy topic[%s]: route meta request from proxy to master failed - %s", t.name, err)
-				msg.sess.queueOut(ErrClusterUnreachableReply(msg, types.TimeNow()))
+				logClusterProxyRouteFailure(t.name, "meta", err)
+				msg.sess.queueOut(clusterRouteErrorReply(msg, err))
 			}
 
 		case upd := <-t.supd:
@@ -122,6 +144,17 @@ func (t *Topic) runProxy(hub *Hub) {
 	}
 }
 
+func clusterRouteErrorReply(msg *ClientComMessage, err error) *ServerComMessage {
+	if errors.Is(err, errClusterLoadExceeded) || errors.Is(err, errClusterOverloaded) {
+		immetrics.ClusterProxyResponseTotal.WithLabelValues(clusterPodName(), "overloaded").Inc()
+		reply := ErrServiceUnavailableReply(msg, types.TimeNow())
+		reply.Ctrl.Text = "cluster overloaded"
+		return reply
+	}
+	immetrics.ClusterProxyResponseTotal.WithLabelValues(clusterPodName(), "unreachable").Inc()
+	return ErrClusterUnreachableReply(msg, types.TimeNow())
+}
+
 // Takes a session leave request, forwards it to the topic master and
 // modifies the local state accordingly.
 // Returns whether the operation was successful.