pro/im_server
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: harden multi-pod delivery and load verification

Browse Source
This commit is contained in:
richjohnrambo
2026-09-27 03:37:32 +08:00
parent a2de100f2b
commit ce3250be64
33 changed files with 1340 additions and 94 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/cluster.go
+167 -34
View File
@@ -4,6 +4,7 @@ import (
"encoding/gob"
"encoding/json"
"errors"
"hash/fnv"
"net"
"net/rpc"
"sort"
@@ -14,6 +15,7 @@ import (
"github.com/tinode/chat/server/auth"
"github.com/tinode/chat/server/concurrency"
"github.com/tinode/chat/server/logs"
immetrics "github.com/tinode/chat/server/metrics"
"github.com/tinode/chat/server/push"
rh "github.com/tinode/chat/server/ringhash"
"github.com/tinode/chat/server/store/types"
@@ -25,18 +27,21 @@ const (
// Default timeout before attempting to reconnect to a node.
clusterDefaultReconnectTime = 200 * time.Millisecond
// Number of replicas in ringhash.
clusterHashReplicas = 20
// Buffer size for sending requests from proxy to master.
clusterProxyToMasterBuffer = 64
// Expand buffer size by this value for nodes over the basic 3-node setup.
clusterProxyToMasterBufferPerNode = 16
// Timeout for attempting to enqueue a proxy-to-master request when the buffer is full.
clusterP2MTimeout = 20 * time.Millisecond
clusterHashReplicas = 20
clusterDefaultP2MBuffer = 4096
clusterDefaultP2MWorkers = 16
clusterDefaultP2MTimeout = 300 * time.Millisecond
clusterDefaultRouteRetries = 3
clusterDefaultRouteRetryBackoff = 25 * time.Millisecond
// Buffer size for receiving responses from other nodes, per node.
clusterRpcCompletionBuffer = 64
)
var errClusterTopicOwnerUnavailable = errors.New("cluster topic owner unavailable")
var (
errClusterTopicOwnerUnavailable = errors.New("cluster topic owner unavailable")
errClusterOverloaded = errors.New("cluster overloaded")
errClusterLoadExceeded = errors.New("cluster load exceeded")
)
// ProxyReqType is the type of proxy requests.
type ProxyReqType int
@@ -64,7 +69,12 @@ type clusterConfig struct {
// Name of this cluster node
ThisName string `json:"self"`
// Deprecated: this field is no longer used.
NumProxyEventGoRoutines int `json:"-"`
NumProxyEventGoRoutines int `json:"-"`
ProxyToMasterBuffer int `json:"proxy_to_master_buffer"`
ProxyToMasterWorkers int `json:"proxy_to_master_workers"`
ProxyToMasterEnqueueTimeoutMS int `json:"proxy_to_master_enqueue_timeout_ms"`
RouteRetries int `json:"route_retries"`
RouteRetryBackoffMS int `json:"route_retry_backoff_ms"`
// Failover configuration
Failover *clusterFailoverConfig
}
@@ -99,8 +109,13 @@ type ClusterNode struct {
// Buffered, clusterRpcCompletionBuffer * number_of_nodes.
rpcDone chan *rpc.Call
// Channel for sending proxy to master requests; buffered, clusterProxyToMasterBuffer.
p2mSender chan *ClusterReq
// Sharded queues preserve ordering for the same tenant and topic while allowing parallel RPCs.
p2mSenders []chan *ClusterReq
p2mEnqueueTimeout time.Duration
routeRetries int
routeRetryBackoff time.Duration
overloadLogAt atomic.Int64
overloadSuppressed atomic.Int64
}
func (n *ClusterNode) asyncRpcLoop() {
@@ -109,19 +124,51 @@ func (n *ClusterNode) asyncRpcLoop() {
}
}
func (n *ClusterNode) p2mSenderLoop() {
for req := range n.p2mSender {
func (n *ClusterNode) p2mSenderLoop(sender <-chan *ClusterReq) {
for req := range sender {
if req == nil {
// Stop
return
}
if err := n.proxyToMaster(req); err != nil {
logs.Warn.Println("p2mSenderLoop: call failed", n.name, err)
if errors.Is(err, errClusterOverloaded) {
n.logOverload("proxy_to_master", req.TenantID, req.RcptTo, 0)
} else {
logs.Warn.Println("p2mSenderLoop: call failed", n.name, err)
}
}
n.updateP2MQueueMetric()
}
}
func (n *ClusterNode) logOverload(path string, tenantID types.TenantID, topic string, seq int64) {
now := time.Now().Unix()
previous := n.overloadLogAt.Load()
if previous == now || !n.overloadLogAt.CompareAndSwap(previous, now) {
n.overloadSuppressed.Add(1)
return
}
suppressed := n.overloadSuppressed.Swap(0)
logs.Warn.Printf("cluster overload: path=%s tenant=%d topic=%s seq=%d peer=%s suppressed=%d",
path, tenantID, topic, seq, n.name, suppressed)
}
func (n *ClusterNode) updateP2MQueueMetric() {
depth := 0
for _, sender := range n.p2mSenders {
depth += len(sender)
}
immetrics.ClusterP2MQueueLength.WithLabelValues(clusterPodName(), n.name).Set(float64(depth))
}
func clusterPodName() string {
if globals.cluster != nil && globals.cluster.thisNodeName != "" {
return globals.cluster.thisNodeName
}
return "standalone"
}
// ClusterSess is a basic info on a remote session where the message was created.
type ClusterSess struct {
TenantID types.TenantID
@@ -318,7 +365,12 @@ func (n *ClusterNode) call(proc string, req, resp any) error {
return errors.New("cluster: node '" + n.name + "' not connected")
}
method := proc
started := time.Now()
immetrics.ClusterRPCInflight.WithLabelValues(clusterPodName(), n.name, method).Inc()
defer immetrics.ClusterRPCInflight.WithLabelValues(clusterPodName(), n.name, method).Dec()
if err := n.endpoint.Call(proc, req, resp); err != nil {
immetrics.ClusterRPCDuration.WithLabelValues(clusterPodName(), n.name, method, "error").Observe(time.Since(started).Seconds())
logs.Warn.Println("cluster: call failed", n.name, err)
n.lock.Lock()
@@ -331,6 +383,7 @@ func (n *ClusterNode) call(proc string, req, resp any) error {
n.lock.Unlock()
return err
}
immetrics.ClusterRPCDuration.WithLabelValues(clusterPodName(), n.name, method, "ok").Observe(time.Since(started).Seconds())
return nil
}
@@ -395,29 +448,47 @@ func (n *ClusterNode) proxyToMaster(msg *ClusterReq) error {
var rejected bool
err := n.call("Cluster.TopicMaster", msg, &rejected)
if err == nil && rejected {
err = errors.New("cluster: topic master node out of sync")
err = errClusterOverloaded
}
return err
}
// proxyToMaster forwards request from topic proxy to topic master.
func (n *ClusterNode) proxyToMasterAsync(msg *ClusterReq) error {
if len(n.p2mSenders) == 0 {
return errClusterTopicOwnerUnavailable
}
sender := n.p2mSenders[clusterReqShard(msg, len(n.p2mSenders))]
select {
case n.p2mSender <- msg:
case sender <- msg:
n.updateP2MQueueMetric()
immetrics.ClusterP2MEnqueueTotal.WithLabelValues(clusterPodName(), n.name, "ok").Inc()
return nil
default:
}
// Buffer is full. Wait briefly before giving up.
timer := time.NewTimer(clusterP2MTimeout)
timer := time.NewTimer(n.p2mEnqueueTimeout)
defer timer.Stop()
select {
case n.p2mSender <- msg:
case sender <- msg:
n.updateP2MQueueMetric()
immetrics.ClusterP2MEnqueueTotal.WithLabelValues(clusterPodName(), n.name, "ok").Inc()
return nil
case <-timer.C:
return errors.New("cluster: load exceeded")
immetrics.ClusterP2MEnqueueTotal.WithLabelValues(clusterPodName(), n.name, "timeout").Inc()
return errClusterLoadExceeded
}
}
func clusterReqShard(msg *ClusterReq, count int) int {
if count <= 1 || msg == nil {
return 0
}
h := fnv.New32a()
_, _ = h.Write([]byte(types.TopicKey{TenantID: msg.TenantID, Topic: msg.RcptTo}.RoutingKey()))
return int(h.Sum32() % uint32(count))
}
// masterToProxyAsync forwards response from topic master to topic proxy
// in a fire-and-forget manner.
func (n *ClusterNode) masterToProxyAsync(msg *ClusterResp) error {
@@ -430,8 +501,32 @@ func (n *ClusterNode) masterToProxyAsync(msg *ClusterResp) error {
// route routes server message within the cluster.
func (n *ClusterNode) route(msg *ClusterRoute) error {
var unused bool
return n.call("Cluster.Route", msg, &unused)
for attempt := 0; attempt <= n.routeRetries; attempt++ {
var rejected bool
err := n.call("Cluster.Route", msg, &rejected)
if err != nil {
immetrics.ClusterRouteTotal.WithLabelValues(clusterPodName(), n.name, "disconnected").Inc()
return err
}
if !rejected {
immetrics.ClusterRouteTotal.WithLabelValues(clusterPodName(), n.name, "ok").Inc()
return nil
}
immetrics.ClusterRouteTotal.WithLabelValues(clusterPodName(), n.name, "peer_rejected").Inc()
if attempt < n.routeRetries {
time.Sleep(n.routeRetryBackoff * time.Duration(1<<attempt))
}
}
seq := int64(0)
topic := ""
if msg != nil && msg.SrvMsg != nil && msg.SrvMsg.Data != nil {
seq = msg.SrvMsg.Data.SeqId
}
if msg != nil && msg.SrvMsg != nil {
topic = msg.SrvMsg.RcptTo
}
n.logOverload("route", msg.TenantID, topic, seq)
return errClusterOverloaded
}
// Cluster is the representation of the cluster.
@@ -460,6 +555,8 @@ type Cluster struct {
// running a separate event processing goroutine for each proxy session
// leads to a rather large memory usage and excessive scheduling overhead.
proxyEventQueue *concurrency.GoRoutinePool
p2mBuffer int
p2mWorkers int
}
func (n *ClusterNode) stopMultiplexingSession(msess *Session) {
@@ -600,7 +697,13 @@ func (c *Cluster) TopicMaster(msg *ClusterReq, rejected *bool) error {
select {
case globals.hub.routeCli <- msg.CliMsg:
default:
logs.Err.Println("cluster: route req failed - hub.route queue full")
*rejected = true
if sess != nil {
reply := ErrServiceUnavailableReply(msg.CliMsg, msg.CliMsg.Timestamp)
reply.Ctrl.Text = "cluster overloaded"
sess.queueOut(reply)
}
node.logOverload("topic_master", msg.TenantID, msg.RcptTo, 0)
}
case ProxyReqBgSession, ProxyReqMeUserAgent:
@@ -681,7 +784,9 @@ func (c *Cluster) Route(msg *ClusterRoute, rejected *bool) error {
select {
case globals.hub.routeSrv <- msg.SrvMsg:
immetrics.ClusterRouteTotal.WithLabelValues(clusterPodName(), msg.Node, "ok").Inc()
default:
immetrics.ClusterRouteTotal.WithLabelValues(clusterPodName(), msg.Node, "hub_full").Inc()
logError("cluster Route: server busy")
}
return nil
@@ -1016,6 +1121,26 @@ func clusterInit(configString json.RawMessage, self *string) int {
if err := json.Unmarshal(configString, &config); err != nil {
logs.Err.Fatal(err)
}
if config.ProxyToMasterBuffer <= 0 {
config.ProxyToMasterBuffer = clusterDefaultP2MBuffer
}
if config.ProxyToMasterWorkers <= 0 {
config.ProxyToMasterWorkers = clusterDefaultP2MWorkers
}
if config.ProxyToMasterWorkers > config.ProxyToMasterBuffer {
config.ProxyToMasterWorkers = config.ProxyToMasterBuffer
}
if config.ProxyToMasterEnqueueTimeoutMS <= 0 {
config.ProxyToMasterEnqueueTimeoutMS = int(clusterDefaultP2MTimeout / time.Millisecond)
}
if config.RouteRetries < 0 {
config.RouteRetries = 0
} else if config.RouteRetries == 0 {
config.RouteRetries = clusterDefaultRouteRetries
}
if config.RouteRetryBackoffMS <= 0 {
config.RouteRetryBackoffMS = int(clusterDefaultRouteRetryBackoff / time.Millisecond)
}
thisName := *self
if thisName == "" {
@@ -1039,6 +1164,8 @@ func clusterInit(configString json.RawMessage, self *string) int {
fingerprint: time.Now().Unix(),
nodes: make(map[string]*ClusterNode),
proxyEventQueue: concurrency.NewGoRoutinePool(len(config.Nodes) * 5),
p2mBuffer: config.ProxyToMasterBuffer,
p2mWorkers: config.ProxyToMasterWorkers,
}
var nodeNames []string
@@ -1052,10 +1179,13 @@ func clusterInit(configString json.RawMessage, self *string) int {
}
globals.cluster.nodes[host.Name] = &ClusterNode{
address: host.Addr,
name: host.Name,
done: make(chan bool, 1),
msess: make(map[string]struct{}),
address: host.Addr,
name: host.Name,
done: make(chan bool, 1),
msess: make(map[string]struct{}),
p2mEnqueueTimeout: time.Duration(config.ProxyToMasterEnqueueTimeoutMS) * time.Millisecond,
routeRetries: config.RouteRetries,
routeRetryBackoff: time.Duration(config.RouteRetryBackoffMS) * time.Millisecond,
}
}
@@ -1113,17 +1243,18 @@ func (c *Cluster) start() {
logs.Err.Fatal(err)
}
var bufferSize = clusterProxyToMasterBuffer
if len(c.nodes) > 2 {
// Expand the buffer for larger (>3 node) clusters.
bufferSize += clusterProxyToMasterBufferPerNode * (len(c.nodes) - 2)
}
for _, n := range c.nodes {
go n.reconnect()
n.rpcDone = make(chan *rpc.Call, len(c.nodes)*clusterRpcCompletionBuffer)
n.p2mSender = make(chan *ClusterReq, bufferSize)
workers := c.p2mWorkers
perWorker := (c.p2mBuffer + workers - 1) / workers
n.p2mSenders = make([]chan *ClusterReq, workers)
go n.asyncRpcLoop()
go n.p2mSenderLoop()
for i := range n.p2mSenders {
n.p2mSenders[i] = make(chan *ClusterReq, perWorker)
go n.p2mSenderLoop(n.p2mSenders[i])
}
immetrics.ClusterP2MQueueCapacity.WithLabelValues(clusterPodName(), n.name).Set(float64(perWorker * workers))
}
if c.fo != nil {
@@ -1147,7 +1278,9 @@ func (c *Cluster) shutdown() {
}
for _, n := range c.nodes {
close(n.rpcDone)
close(n.p2mSender)
for _, sender := range n.p2mSenders {
close(sender)
}
}
globals.cluster.proxyEventQueue.Stop()