pro/im_server
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: harden message seq allocation against stale redis replicas

Browse Source 
- use sentinel failover client for group message runtime when
  sentinel_master_name/sentinel_addrs are configured, so seq counter
  reads never hit a stale replica after failover
- re-seed missing redis seq counter from DB topic seqid instead of
  trusting the caller in-memory baseline
- on unique-key conflict during async persist, re-allocate seq from DB
  latest, bump redis counter and fix the idempotency record so resends
  with the same client mid return the final seq
- add uk(topic, seqid) unique keys for im_messages in tenant DDL
This commit is contained in:
Austin
2026-09-15 12:41:38 +08:00
parent 4f6a996922
commit 5dd0a42002
4 changed files with 205 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/store/store.go
+55 -20
View File
@@ -3597,7 +3597,7 @@ func (messagesMapper) PreparePersist(tenantID types.TenantID, msg *types.Message
return nil, types.ErrDuplicate
}
}
seq, err := groupMsgRuntime.NextSeq(tenantID, msg.Topic, msg.SeqId-1)
seq, err := nextMessageSeqWithFloor(tenantID, msg.Topic, msg.SeqId-1)
if err != nil {
if clientMsgID != "" {
_ = groupMsgRuntime.MarkFailed(tenantID, msg.Topic, fromUid, clientMsgID)
@@ -3661,21 +3661,21 @@ func (messagesMapper) PersistPrepared(tenantID types.TenantID, msg *types.Messag
err = tenantAdp.TenantMessageSave(tenantID, msg)
}
if err != nil {
if isGroupMessage && err == types.ErrDuplicate && clientMsgID != "" {
duplicate, dupErr := tenantAdp.TenantGroupMessageGetByClientID(tenantID, msg.Topic, fromUid, clientMsgID)
if dupErr != nil {
return nil, dupErr
}
if duplicate != nil {
msg.SeqId = duplicate.SeqId
if groupMsgRuntime != nil {
_ = groupMsgRuntime.MarkDone(tenantID, msg.Topic, fromUid, clientMsgID, duplicate.SeqId, duplicate.Uid().String())
if err == types.ErrDuplicate {
if isGroupMessage && clientMsgID != "" {
duplicate, dupErr := tenantAdp.TenantGroupMessageGetByClientID(tenantID, msg.Topic, fromUid, clientMsgID)
if dupErr != nil {
return nil, dupErr
}
if duplicate != nil {
msg.SeqId = duplicate.SeqId
if groupMsgRuntime != nil {
_ = groupMsgRuntime.MarkDone(tenantID, msg.Topic, fromUid, clientMsgID, duplicate.SeqId, duplicate.Uid().String())
}
return &types.MessageSaveResult{SeqId: duplicate.SeqId, Duplicate: true}, nil
}
return &types.MessageSaveResult{SeqId: duplicate.SeqId, Duplicate: true}, nil
}
}
if !isGroupMessage && err == types.ErrDuplicate {
if clientMsgID != "" {
if !isGroupMessage && clientMsgID != "" {
duplicate, dupErr := tenantAdp.TenantMessageGetByClientID(tenantID, fromUid, clientMsgID)
if dupErr != nil {
return nil, dupErr
@@ -3685,12 +3685,14 @@ func (messagesMapper) PersistPrepared(tenantID types.TenantID, msg *types.Messag
return &types.MessageSaveResult{SeqId: duplicate.SeqId, Duplicate: true}, nil
}
}
if retryErr := retryTenantMessageSaveAfterSeqConflict(tenantAdp, tenantID, msg); retryErr != nil {
if retryErr := retryTenantMessageSaveAfterSeqConflict(tenantAdp, tenantID, msg, isGroupMessage); retryErr != nil {
if isGroupMessage && groupMsgRuntime != nil && clientMsgID != "" {
_ = groupMsgRuntime.MarkFailed(tenantID, msg.Topic, fromUid, clientMsgID)
}
return nil, retryErr
}
err = nil
}
if isGroupMessage && groupMsgRuntime != nil && clientMsgID != "" {
} else if isGroupMessage && groupMsgRuntime != nil && clientMsgID != "" {
_ = groupMsgRuntime.MarkFailed(tenantID, msg.Topic, fromUid, clientMsgID)
}
if err != nil {
@@ -3786,19 +3788,52 @@ func NormalizeClientMsgID(head types.KVMap) (string, error) {
return normalizeClientMsgID(head)
}
func retryTenantMessageSaveAfterSeqConflict(tenantAdp adapter.TenantBusinessAdapter, tenantID types.TenantID, msg *types.Message) error {
latest, err := tenantAdp.TenantMessageGetAll(tenantID, msg.Topic, types.ZeroUid, &types.QueryOpt{Before: 1 << 62, Limit: 1})
func retryTenantMessageSaveAfterSeqConflict(tenantAdp adapter.TenantBusinessAdapter, tenantID types.TenantID, msg *types.Message, isGroupMessage bool) error {
var latest []types.Message
var err error
if isGroupMessage {
latest, err = tenantAdp.TenantGroupMessageGetAll(tenantID, msg.Topic, types.ZeroUid, &types.QueryOpt{Before: 1 << 62, Limit: 1})
} else {
latest, err = tenantAdp.TenantMessageGetAll(tenantID, msg.Topic, types.ZeroUid, &types.QueryOpt{Before: 1 << 62, Limit: 1})
}
if err != nil {
return err
}
if len(latest) == 0 || latest[0].SeqId < msg.SeqId {
return types.ErrDuplicate
}
prevSeq := msg.SeqId
msg.SeqId = latest[0].SeqId + 1
if err = tenantAdp.TenantTopicUpdateOnMessage(tenantID, msg.Topic, msg); err != nil {
return err
}
return tenantAdp.TenantMessageSave(tenantID, msg)
if isGroupMessage {
err = tenantAdp.TenantGroupMessageSave(tenantID, msg)
} else {
err = tenantAdp.TenantMessageSave(tenantID, msg)
}
if err != nil {
return err
}
// The re-allocated seq differs from the one handed out before the insert
// failed. Sync the Redis allocator and the idempotency record with the
// final seq, otherwise subsequent publishes keep colliding with the
// persisted row and resends of the same client mid return the abandoned
// seq forever.
BumpMessageSeqFloor(tenantID, msg.Topic, msg.SeqId)
if isGroupMessage {
// Drop the pending hot-cache entry the producer wrote for the
// abandoned seq so hot-range reads cannot surface a phantom message.
if err := InvalidateHotMessages(tenantID, msg.Topic, []types.Range{{Low: prevSeq, Hi: prevSeq + 1}}); err != nil {
logs.Warn.Printf("topic[%s]: failed to invalidate abandoned hot message seq=%d: %v", msg.Topic, prevSeq, err)
}
}
if msg.ClientMsgID != "" {
if fromUid := types.ParseUid(msg.From); !fromUid.IsZero() {
_ = MarkMessageUniqueDone(tenantID, msg.Topic, fromUid, msg.ClientMsgID, msg.SeqId, msg.Uid().String())
}
}
return nil
}
func normalizeClientMsgID(head types.KVMap) (string, error) {