pro/im_server
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: shard group messages by month

Browse Source
This commit is contained in:
richjohnrambo
2026-09-15 23:53:51 +08:00
parent b7c2536527
commit 139681fadb
25 changed files with 1718 additions and 247 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/kafka_message_persist.go
+19 -3
View File
@@ -18,6 +18,7 @@ type kafkaMessagePersistPayload struct {
AttachmentURLs []string `json:"attachment_urls,omitempty"`
ReadBySender bool `json:"read_by_sender,omitempty"`
MessageKind string `json:"message_kind"`
ShardMonth string `json:"shard_month,omitempty"`
}
var publishKafkaMessagePersist = publishKafkaEnvelope
@@ -34,6 +35,11 @@ func saveMessageMaybeKafka(ctx context.Context, tenantID types.TenantID, msg *ty
if msg == nil || msg.TenantID != tenantID || msg.Topic == "" || msg.SeqId <= 0 {
return nil, types.ErrMalformed
}
msg.InitTimes()
if msg.Uid().IsZero() {
msg.SetUid(store.Store.GetUid())
}
isGroupMessage := types.GetTopicCat(msg.Topic) == types.TopicCatGrp
clientMsgID, err := store.NormalizeClientMsgID(msg.Head)
if err != nil {
return nil, err
@@ -56,15 +62,21 @@ func saveMessageMaybeKafka(ctx context.Context, tenantID types.TenantID, msg *ty
return nil, types.ErrDuplicate
}
}
if seq, runtimeReady, seqErr := store.ReserveMessageSeq(tenantID, msg.Topic, msg.SeqId-1); seqErr != nil {
candidateMonth := msg.CreatedAt.UTC().Format("200601")
if seq, shardMonth, runtimeReady, seqErr := store.ReserveMessageSeq(tenantID, msg.Topic, msg.SeqId-1, candidateMonth); seqErr != nil {
if clientMsgID != "" {
_ = store.MarkMessageUniqueFailed(tenantID, msg.Topic, fromUid, clientMsgID)
}
return nil, seqErr
} else if runtimeReady {
msg.SeqId = seq
if isGroupMessage {
msg.ShardMonth = shardMonth
}
}
if msg.ShardMonth == "" && isGroupMessage {
msg.ShardMonth = candidateMonth
}
msg.InitTimes()
if err = enqueueKafkaMessagePersist(ctx, tenantID, msg, attachmentURLs, readBySender); err != nil {
if clientMsgID != "" {
if markErr := store.MarkMessageUniquePending(tenantID, msg.Topic, fromUid, clientMsgID, msg.SeqId); markErr != nil {
@@ -79,7 +91,7 @@ func saveMessageMaybeKafka(ctx context.Context, tenantID types.TenantID, msg *ty
logs.Warn.Printf("kafka message persist: failed to mark message unique key done topic=%s seq=%d: %v", msg.Topic, msg.SeqId, err)
}
}
if types.GetTopicCat(msg.Topic) == types.TopicCatGrp {
if isGroupMessage {
if cacheErr := store.CachePendingHotMessage(tenantID, msg); cacheErr != nil {
logs.Warn.Printf("kafka message persist: failed to cache pending hot message after publish topic=%s seq=%d: %v", msg.Topic, msg.SeqId, cacheErr)
}
@@ -97,6 +109,7 @@ func enqueueKafkaMessagePersist(ctx context.Context, tenantID types.TenantID, ms
AttachmentURLs: attachmentURLs,
ReadBySender: readBySender,
MessageKind: kind,
ShardMonth: msg.ShardMonth,
})
if err != nil {
return err
@@ -147,6 +160,9 @@ func handleKafkaMessagePersistFromTopic(ctx context.Context, envelope kafkaEnvel
if payload.Message.TenantID == types.ZeroTenantID {
payload.Message.TenantID = tenantID
}
if payload.Message.ShardMonth == "" {
payload.Message.ShardMonth = payload.ShardMonth
}
if payload.Message.TenantID != tenantID {
return markKafkaDeadLetter(ctx, tenantID, envelope, consume, "TENANT_MISMATCH", "message tenant does not match envelope tenant")
}