pro/im_server
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

perf: expose and reduce message publish latency

Browse Source
This commit is contained in:
richjohnrambo
2026-09-27 21:08:35 +08:00
parent 8e778726e0
commit bde02e8eac
15 changed files with 274 additions and 98 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/kafka_message_persist.go
+23 -6
View File
@@ -53,7 +53,9 @@ func saveMessageMaybeKafka(ctx context.Context, tenantID types.TenantID, msg *ty
msg.ClientMsgID = clientMsgID
fromUid := types.ParseUid(msg.From)
if clientMsgID != "" {
started := time.Now()
claim, claimed, runtimeReady, claimErr := store.ClaimMessageUnique(tenantID, msg.Topic, fromUid, clientMsgID)
immetrics.ObserveMessagePublishStage(messageKind(msg.Topic), "redis_idempotency", started, claimErr)
if claimErr != nil {
return nil, claimErr
}
@@ -69,12 +71,16 @@ func saveMessageMaybeKafka(ctx context.Context, tenantID types.TenantID, msg *ty
}
}
candidateMonth := msg.CreatedAt.UTC().Format("200601")
if seq, shardMonth, runtimeReady, seqErr := store.ReserveMessageSeq(tenantID, msg.Topic, msg.SeqId-1, candidateMonth); seqErr != nil {
seqStarted := time.Now()
seq, shardMonth, runtimeReady, seqErr := store.ReserveMessageSeq(tenantID, msg.Topic, msg.SeqId-1, candidateMonth)
immetrics.ObserveMessagePublishStage(messageKind(msg.Topic), "redis_sequence", seqStarted, seqErr)
if seqErr != nil {
if clientMsgID != "" {
_ = store.MarkMessageUniqueFailed(tenantID, msg.Topic, fromUid, clientMsgID)
}
return nil, seqErr
} else if runtimeReady {
}
if runtimeReady {
msg.SeqId = seq
if isGroupMessage {
msg.ShardMonth = shardMonth
@@ -83,7 +89,9 @@ func saveMessageMaybeKafka(ctx context.Context, tenantID types.TenantID, msg *ty
if msg.ShardMonth == "" && isGroupMessage {
msg.ShardMonth = candidateMonth
}
kafkaStarted := time.Now()
if err = enqueueKafkaMessagePersist(ctx, tenantID, msg, attachmentURLs, readBySender); err != nil {
immetrics.ObserveMessagePublishStage(messageKind(msg.Topic), "kafka_produce", kafkaStarted, err)
if clientMsgID != "" {
if markErr := store.MarkMessageUniquePending(tenantID, msg.Topic, fromUid, clientMsgID, msg.SeqId); markErr != nil {
logs.Warn.Printf("kafka message persist: failed to mark message unique key pending topic=%s seq=%d: %v", msg.Topic, msg.SeqId, markErr)
@@ -92,6 +100,7 @@ func saveMessageMaybeKafka(ctx context.Context, tenantID types.TenantID, msg *ty
logs.Warn.Printf("kafka message persist: publish state unknown topic=%s seq=%d client_mid=%s err=%v", msg.Topic, msg.SeqId, clientMsgID, err)
return &types.MessageSaveResult{SeqId: msg.SeqId, Pending: true, MarkedReadBySender: readBySender && !fromUid.IsZero()}, err
}
immetrics.ObserveMessagePublishStage(messageKind(msg.Topic), "kafka_produce", kafkaStarted, nil)
if clientMsgID != "" {
if err = store.MarkMessageUniquePending(tenantID, msg.Topic, fromUid, clientMsgID, msg.SeqId); err != nil {
logs.Warn.Printf("kafka message persist: failed to mark message unique key pending topic=%s seq=%d: %v", msg.Topic, msg.SeqId, err)
@@ -105,11 +114,19 @@ func saveMessageMaybeKafka(ctx context.Context, tenantID types.TenantID, msg *ty
return &types.MessageSaveResult{SeqId: msg.SeqId, Pending: true, MarkedReadBySender: readBySender && !fromUid.IsZero()}, nil
}
func enqueueKafkaMessagePersist(ctx context.Context, tenantID types.TenantID, msg *types.Message, attachmentURLs []string, readBySender bool) error {
kind := "p2p"
if types.GetTopicCat(msg.Topic) == types.TopicCatGrp {
kind = "group"
func messageKind(topic string) string {
switch types.GetTopicCat(topic) {
case types.TopicCatGrp:
return "group"
case types.TopicCatP2P:
return "p2p"
default:
return "other"
}
}
func enqueueKafkaMessagePersist(ctx context.Context, tenantID types.TenantID, msg *types.Message, attachmentURLs []string, readBySender bool) error {
kind := messageKind(msg.Topic)
payload, err := json.Marshal(kafkaMessagePersistPayload{
Message: *msg,
AttachmentURLs: attachmentURLs,
server/main.go
+5
View File
@@ -182,6 +182,8 @@ var globals struct {
tlsRedirectHTTP string
// Maximum message size allowed from peer.
maxMessageSize int64
// Log complete client packets. Disabled by default because it is expensive on the publish hot path.
logClientPackets bool
// Maximum number of group topic subscribers.
maxSubscriberCount int
// Maximum number of indexable tags.
@@ -335,6 +337,8 @@ type configType struct {
// Maximum message size allowed from client. Intended to prevent malicious client from sending
// very large files inband (does not affect out of band uploads).
MaxMessageSize int `json:"max_message_size"`
// Log every decoded client packet. Intended only for short diagnostic sessions.
LogClientPackets bool `json:"log_client_packets"`
// Maximum number of group topic subscribers.
MaxSubscriberCount int `json:"max_subscriber_count"`
// Masked tags namespaces: tags immutable on User (mask), mutable on Topic only within the mask.
@@ -670,6 +674,7 @@ func main() {
if globals.maxMessageSize <= 0 {
globals.maxMessageSize = defaultMaxMessageSize
}
globals.logClientPackets = config.LogClientPackets
// Maximum number of group topic subscribers
globals.maxSubscriberCount = config.MaxSubscriberCount
if globals.maxSubscriberCount <= 1 {
server/meetingim.conf
+1
View File
@@ -10,6 +10,7 @@
"api_key_salt": "T713/rYYgW7g4m3vG6zGRh7+FM1t0T8j13koXScOAj4=",
"max_message_size": 131072,
"log_client_packets": false,
"max_subscriber_count": 128,
"max_tag_count": 16,
"permanent_accounts": false,
server/metrics/metrics.go
+16 -1
View File
@@ -107,6 +107,11 @@ var (
Help: "Kafka message persistence duration by message kind and result.",
Buckets: []float64{0.001, 0.003, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10},
}, []string{"kind", "result"})
MessagePublishStageDuration = prometheus.NewHistogramVec(prometheus.HistogramOpts{
Name: "im_message_publish_stage_duration_seconds",
Help: "Message publish hot-path duration by message kind, stage, and result.",
Buckets: []float64{0.001, 0.003, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10},
}, []string{"kind", "stage", "result"})
KafkaMessagePersistTotal = prometheus.NewCounterVec(prometheus.CounterOpts{
Name: "im_kafka_message_persist_total",
Help: "Kafka message persistence outcomes by message kind and result.",
@@ -130,7 +135,8 @@ func init() {
HTTPRequestsTotal, HTTPRequestDuration, DBQueryDuration,
RealtimeFanoutEventsTotal, RealtimeFanoutDeliveryDuration, ClusterP2MQueueLength,
ClusterP2MQueueCapacity, ClusterP2MEnqueueTotal, ClusterRPCInflight, ClusterRPCDuration,
ClusterRouteTotal, ClusterProxyResponseTotal, MessagePersistDuration, KafkaMessagePersistTotal)
ClusterRouteTotal, ClusterProxyResponseTotal, MessagePersistDuration, MessagePublishStageDuration,
KafkaMessagePersistTotal)
Registry.MustRegister(KafkaConsumerLag, KafkaBacklogOldestAge, KafkaDeadLettersTotal)
}
@@ -149,6 +155,15 @@ func ObserveDB(driverName, operation string, started time.Time, err error) {
DBQueryDuration.WithLabelValues(driverName, operation, status).Observe(time.Since(started).Seconds())
}
func ObserveMessagePublishStage(kind, stage string, started time.Time, err error) {
status := "ok"
if err != nil {
status = "error"
}
MessagePublishStageDuration.WithLabelValues(normalizeLabel(kind), normalizeLabel(stage), status).
Observe(time.Since(started).Seconds())
}
func IncRealtimeFanout(stage, kind, targetType, status string) {
RealtimeFanoutEventsTotal.WithLabelValues(normalizeLabel(stage), normalizeLabel(kind),
normalizeLabel(targetType), normalizeLabel(status)).Inc()
server/session.go
+8 -6
View File
@@ -519,13 +519,15 @@ func (s *Session) dispatchRaw(raw []byte) {
s.queueOut(ErrMalformed("", "", now))
return
}
toLog := redactClientMessageForLog(raw)
truncated := ""
if len(toLog) > 512 {
toLog = toLog[:512]
truncated = "<...>"
if globals.logClientPackets {
toLog := redactClientMessageForLog(raw)
truncated := ""
if len(toLog) > 512 {
toLog = toLog[:512]
truncated = "<...>"
}
logs.Info.Printf("in: '%s%s' sid='%s' uid='%s'", toLog, truncated, s.sid, s.uid)
}
logs.Info.Printf("in: '%s%s' sid='%s' uid='%s'", toLog, truncated, s.sid, s.uid)
s.dispatch(&msg)
}
server/store/store.go
+1 -1
View File
@@ -3373,7 +3373,7 @@ func relationshipContext(tenantAdp adapter.TenantBusinessAdapter, tenantID types
if friendship, err := tenantAdp.TenantFriendshipGet(tenantID, userID, peerID); err != nil {
return nil, err
} else if friendship != nil && friendship.Status == types.RelationshipFriend {
cacheFriendOneWay(tenantID, userID, peerID)
cacheFriendPair(tenantID, userID, peerID)
ctx.Relationship = types.RelationshipFriend
ctx.CanSend = true
ctx.Remark = friendship.Remark
server/topic.go
+11
View File
@@ -20,6 +20,7 @@ import (
"github.com/tinode/chat/server/auth"
"github.com/tinode/chat/server/logs"
immetrics "github.com/tinode/chat/server/metrics"
"github.com/tinode/chat/server/moderation"
"github.com/tinode/chat/server/store"
"github.com/tinode/chat/server/store/types"
@@ -1362,7 +1363,9 @@ func (t *Topic) saveAndBroadcastMessage(msg *ClientComMessage, asUid types.Uid,
store.Relations != nil && store.Store != nil && store.Store.IsOpen() {
peer := t.p2pOtherUser(asUid)
if !peer.IsZero() {
started := time.Now()
canSend, _, err := store.Relations.CanSendP2P(t.tenantID, asUid, peer)
immetrics.ObserveMessagePublishStage("p2p", "relationship_check", started, err)
if err != nil {
logs.Warn.Printf("topic[%s]: failed to check p2p relationship: %v", t.name, err)
msg.sess.queueOut(ErrUnknown(msg.Id, t.original(asUid), msg.Timestamp))
@@ -1375,6 +1378,7 @@ func (t *Topic) saveAndBroadcastMessage(msg *ClientComMessage, asUid types.Uid,
}
}
persistStarted := time.Now()
saveResult, err := saveMessageMaybeKafka(context.Background(), t.tenantID,
&types.Message{
ObjHeader: types.ObjHeader{CreatedAt: msg.Timestamp},
@@ -1385,6 +1389,13 @@ func (t *Topic) saveAndBroadcastMessage(msg *ClientComMessage, asUid types.Uid,
Head: head,
Content: content,
}, attachments, (pud.modeGiven & pud.modeWant).IsReader())
kind := "other"
if t.cat == types.TopicCatP2P {
kind = "p2p"
} else if t.cat == types.TopicCatGrp {
kind = "group"
}
immetrics.ObserveMessagePublishStage(kind, "save", persistStarted, err)
if err != nil {
logs.Warn.Printf("topic[%s]: failed to save message: %v", t.name, err)
msg.sess.queueOut(ErrUnknown(msg.Id, t.original(asUid), msg.Timestamp))