pro/im_server
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix: harden high-fanout delivery under load

Browse Source
This commit is contained in:
richjohnrambo
2026-09-30 23:04:54 +08:00
parent 2b631ab176
commit 19584d0939
14 changed files with 491 additions and 90 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/db/mysql/adapter.go
+21
View File
@@ -4836,6 +4836,27 @@ func isDupe(err error) bool {
return ok && myerr.Number == 1062
}
func isInvalidConnection(err error) bool {
return errors.Is(err, ms.ErrInvalidConn)
}
// retryInvalidConnection retries only operations which are independently
// idempotent. Callers must not use it for unconstrained writes.
func retryInvalidConnection(operation string, fn func() error) error {
err := fn()
if !isInvalidConnection(err) {
return err
}
immetrics.IncDBInvalidConnectionRetry(operation, "attempt")
err = fn()
if err == nil || isDupe(err) {
immetrics.IncDBInvalidConnectionRetry(operation, "recovered")
} else {
immetrics.IncDBInvalidConnectionRetry(operation, "failed")
}
return err
}
func isMissingTable(err error) bool {
if err == nil {
return false
server/db/mysql/invalid_connection_test.go
+55
View File
@@ -0,0 +1,55 @@
//go:build mysql
package mysql
import (
"context"
"database/sql/driver"
"errors"
"testing"
ms "github.com/go-sql-driver/mysql"
)
func TestRetryInvalidConnectionRetriesOnce(t *testing.T) {
attempts := 0
err := retryInvalidConnection("test", func() error {
attempts++
if attempts == 1 {
return ms.ErrInvalidConn
}
return nil
})
if err != nil || attempts != 2 {
t.Fatalf("err=%v attempts=%d", err, attempts)
}
}
func TestRetryInvalidConnectionDoesNotRetryOtherErrors(t *testing.T) {
want := errors.New("write failed")
attempts := 0
err := retryInvalidConnection("test", func() error {
attempts++
return want
})
if !errors.Is(err, want) || attempts != 1 {
t.Fatalf("err=%v attempts=%d", err, attempts)
}
}
func TestInstrumentedQueryDiscardsInvalidConnection(t *testing.T) {
conn := instrumentedConn{conn: invalidQueryConn{}}
_, err := conn.QueryContext(context.Background(), "SELECT 1", nil)
if !errors.Is(err, driver.ErrBadConn) {
t.Fatalf("err=%v want=%v", err, driver.ErrBadConn)
}
}
type invalidQueryConn struct{}
func (invalidQueryConn) Prepare(string) (driver.Stmt, error) { return nil, driver.ErrSkip }
func (invalidQueryConn) Close() error { return nil }
func (invalidQueryConn) Begin() (driver.Tx, error) { return nil, driver.ErrSkip }
func (invalidQueryConn) QueryContext(context.Context, string, []driver.NamedValue) (driver.Rows, error) {
return nil, ms.ErrInvalidConn
}
server/db/mysql/metrics_driver.go
+10
View File
@@ -113,6 +113,12 @@ func (c instrumentedConn) QueryContext(ctx context.Context, query string, args [
started := time.Now()
rows, err := queryer.QueryContext(ctx, query, args)
immetrics.ObserveDB("mysql", "query", started, err)
if isInvalidConnection(err) {
// Queries are idempotent. Returning ErrBadConn makes database/sql discard
// this connection and transparently retry on a fresh one.
immetrics.IncDBInvalidConnectionRetry("query", "discard")
return nil, driver.ErrBadConn
}
return rows, err
}
@@ -211,6 +217,10 @@ func (s instrumentedStmt) QueryContext(ctx context.Context, args []driver.NamedV
started := time.Now()
rows, err := queryer.QueryContext(ctx, args)
immetrics.ObserveDB("mysql", "query", started, err)
if isInvalidConnection(err) {
immetrics.IncDBInvalidConnectionRetry("prepared_query", "discard")
return nil, driver.ErrBadConn
}
return rows, err
}
server/db/mysql/tenant_adapter.go
+30 -15
View File
@@ -2233,21 +2233,29 @@ func (a *adapter) TenantMessageSave(id t.TenantID, msg *t.Message) error {
if a.tenantScopedDB {
return a.MessageSave(msg)
}
ctx, cancel := a.getContext()
if cancel != nil {
defer cancel()
save := func() error {
ctx, cancel := a.getContext()
if cancel != nil {
defer cancel()
}
clientMsgID := sql.NullString{String: msg.ClientMsgID, Valid: msg.ClientMsgID != ""}
res, saveErr := a.db.ExecContext(ctx,
fmt.Sprintf("INSERT INTO %s(tenant_id,createdat,updatedat,seqid,topic,`from`,client_msg_id,version,head,content,content_nonce,key_version) VALUES(?,?,?,?,?,?,?,?,?,?,?,?)", tenantMessagesTable(id)),
id, msg.CreatedAt, msg.UpdatedAt, msg.SeqId, msg.Topic, decodeUID(t.ParseUid(msg.From)), clientMsgID, messageVersion(msg.Version), msg.Head, common.ToJSON(msg.Content), msg.ContentNonce, msg.KeyVersion)
if saveErr == nil {
lastID, _ := res.LastInsertId()
msg.SetUid(t.Uid(lastID))
}
return saveErr
}
if msg.ClientMsgID == "" {
err = save()
} else {
err = retryInvalidConnection("tenant_message_save", save)
}
clientMsgID := sql.NullString{String: msg.ClientMsgID, Valid: msg.ClientMsgID != ""}
res, err := a.db.ExecContext(ctx,
fmt.Sprintf("INSERT INTO %s(tenant_id,createdat,updatedat,seqid,topic,`from`,client_msg_id,version,head,content,content_nonce,key_version) VALUES(?,?,?,?,?,?,?,?,?,?,?,?)", tenantMessagesTable(id)),
id, msg.CreatedAt, msg.UpdatedAt, msg.SeqId, msg.Topic, decodeUID(t.ParseUid(msg.From)), clientMsgID, messageVersion(msg.Version), msg.Head, common.ToJSON(msg.Content), msg.ContentNonce, msg.KeyVersion)
if isDupe(err) {
return t.ErrDuplicate
}
if err == nil {
lastID, _ := res.LastInsertId()
msg.SetUid(t.Uid(lastID))
}
return err
}
@@ -2349,7 +2357,17 @@ func (a *adapter) TenantGroupMessageSave(id t.TenantID, msg *t.Message) error {
if tenantAdp != a {
return tenantAdp.TenantGroupMessageSave(id, msg)
}
if err = a.ensureCurrentGroupMessageShardWindow(); err != nil {
err = retryInvalidConnection("tenant_group_message_save", func() error {
return a.tenantGroupMessageSaveOnce(id, msg)
})
if isDupe(err) {
return t.ErrDuplicate
}
return err
}
func (a *adapter) tenantGroupMessageSaveOnce(id t.TenantID, msg *t.Message) error {
if err := a.ensureCurrentGroupMessageShardWindow(); err != nil {
return err
}
ctx, cancel := a.getContext()
@@ -2398,9 +2416,6 @@ func (a *adapter) TenantGroupMessageSave(id t.TenantID, msg *t.Message) error {
id, messageID, msg.Topic, groupID, msg.SeqId, decodeUID(t.ParseUid(msg.From)), clientMsgID, messageHeadType(msg.Head), "normal",
contentJSON, msg.Head, messageHeadJSON(msg.Head, "mentioned_user_ids"), messageHeadBool(msg.Head, "mention_all"),
0, len(contentJSON), messageVersion(msg.Version), msg.CreatedAt, msg.UpdatedAt, msg.ContentNonce, msg.KeyVersion)
if isDupe(err) {
return t.ErrDuplicate
}
if err != nil {
return err
}
server/hdl_websock.go
+19 -5
View File
@@ -70,14 +70,23 @@ func (sess *Session) readLoop() {
}()
sess.ws.SetReadLimit(globals.maxMessageSize)
sess.ws.SetReadDeadline(time.Now().Add(pongWait))
sess.ws.SetPongHandler(func(string) error {
refreshReadDeadline := func(event, presenceSource string) {
atomic.StoreInt64(&sess.lastAction, types.TimeNow().UnixNano())
immetrics.IncWSHeartbeat("pong_received")
sess.refreshPresence("ws.pong")
sess.ws.SetReadDeadline(time.Now().Add(pongWait))
immetrics.IncWSHeartbeat(event)
if presenceSource != "" {
sess.refreshPresence(presenceSource)
}
_ = sess.ws.SetReadDeadline(time.Now().Add(pongWait))
}
_ = sess.ws.SetReadDeadline(time.Now().Add(pongWait))
sess.ws.SetPongHandler(func(string) error {
refreshReadDeadline("pong_received", "ws.pong")
return nil
})
sess.ws.SetPingHandler(func(appData string) error {
refreshReadDeadline("ping_received", "")
return sess.ws.WriteControl(websocket.PongMessage, []byte(appData), time.Now().Add(writeWait))
})
for {
// Read a ClientComMessage
@@ -90,6 +99,11 @@ func (sess *Session) readLoop() {
}
return
}
// Any successfully read client frame proves that the connection is alive.
// This prevents a busy client from timing out merely because its Pong was
// delayed behind application traffic.
atomic.StoreInt64(&sess.lastAction, types.TimeNow().UnixNano())
_ = sess.ws.SetReadDeadline(time.Now().Add(pongWait))
statsInc("IncomingMessagesWebsockTotal", 1)
sess.dispatchRaw(raw)
}
server/metrics/metrics.go
+9 -1
View File
@@ -29,6 +29,10 @@ var (
Help: "Database operation duration by driver, operation, and status.",
Buckets: []float64{0.001, 0.003, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5},
}, []string{"driver", "operation", "status"})
DBInvalidConnectionRetryTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
Name: "im_db_invalid_connection_retry_total",
Help: "Retries after an invalid database connection by operation and outcome.",
}, []string{"operation", "outcome"})
RealtimeFanoutEventsTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
Name: "realtime_fanout_events_total",
@@ -228,7 +232,7 @@ func init() {
IMWSSendQueueCurrent, IMWSSendQueueCapacity,
IMWSSendQueueEnqueueTotal, IMWSSendQueueDepth, IMWSSendQueueWaitDuration,
IMClusterLivePeers, IMClusterExpectedPeers,
HTTPRequestsTotal, HTTPRequestDuration, DBQueryDuration,
HTTPRequestsTotal, HTTPRequestDuration, DBQueryDuration, DBInvalidConnectionRetryTotal,
RealtimeFanoutEventsTotal, RealtimeFanoutDeliveryDuration, ClusterP2MQueueLength,
ClusterP2MQueueCapacity, ClusterP2MEnqueueTotal, ClusterRPCInflight, ClusterRPCDuration,
ClusterRouteTotal, ClusterProxyResponseTotal, InternalQueueEnqueueTotal, InternalQueueDepth,
@@ -345,6 +349,10 @@ func IncWSHeartbeat(event string) {
IMWSHeartbeatTotal.WithLabelValues(metricPodName(), normalizeLabel(event)).Inc()
}
func IncDBInvalidConnectionRetry(operation, outcome string) {
DBInvalidConnectionRetryTotal.WithLabelValues(normalizeLabel(operation), normalizeLabel(outcome)).Inc()
}
func SetWSHeartbeatConfig(pingPeriod, pongWait time.Duration) {
pod := metricPodName()
IMWSHeartbeatSeconds.WithLabelValues(pod, "ping_period").Set(pingPeriod.Seconds())