Fix Kafka message seq idempotency

server/kafka_message_persist.go

@@ -39,16 +39,46 @@ func saveMessageMaybeKafka(ctx context.Context, tenantID types.TenantID, msg *ty
 		return nil, err
 	}
 	msg.ClientMsgID = clientMsgID
+	fromUid := types.ParseUid(msg.From)
+	if clientMsgID != "" {
+		claim, claimed, runtimeReady, claimErr := store.ClaimMessageUnique(tenantID, msg.Topic, fromUid, clientMsgID)
+		if claimErr != nil {
+			return nil, claimErr
+		}
+		if runtimeReady && !claimed {
+			if claim != nil && claim.Seq > 0 {
+				msg.SeqId = claim.Seq
+				return &types.MessageSaveResult{SeqId: claim.Seq, Duplicate: true}, nil
+			}
+			return nil, types.ErrDuplicate
+		}
+	}
+	if seq, runtimeReady, seqErr := store.ReserveMessageSeq(tenantID, msg.Topic, msg.SeqId-1); seqErr != nil {
+		if clientMsgID != "" {
+			_ = store.MarkMessageUniqueFailed(tenantID, msg.Topic, fromUid, clientMsgID)
+		}
+		return nil, seqErr
+	} else if runtimeReady {
+		msg.SeqId = seq
+	}
 	msg.InitTimes()
 	if err = enqueueKafkaMessagePersist(ctx, tenantID, msg, attachmentURLs, readBySender); err != nil {
+		if clientMsgID != "" {
+			_ = store.MarkMessageUniqueFailed(tenantID, msg.Topic, fromUid, clientMsgID)
+		}
 		return nil, err
 	}
+	if clientMsgID != "" {
+		if err = store.MarkMessageUniqueDone(tenantID, msg.Topic, fromUid, clientMsgID, msg.SeqId, ""); err != nil {
+			logs.Warn.Printf("kafka message persist: failed to mark message unique key done topic=%s seq=%d: %v", msg.Topic, msg.SeqId, err)
+		}
+	}
 	if types.GetTopicCat(msg.Topic) == types.TopicCatGrp {
 		if cacheErr := store.CachePendingHotMessage(tenantID, msg); cacheErr != nil {
 			logs.Warn.Printf("kafka message persist: failed to cache pending hot message after publish topic=%s seq=%d: %v", msg.Topic, msg.SeqId, cacheErr)
 		}
 	}
-	return &types.MessageSaveResult{SeqId: msg.SeqId, MarkedReadBySender: readBySender && !types.ParseUid(msg.From).IsZero()}, nil
+	return &types.MessageSaveResult{SeqId: msg.SeqId, MarkedReadBySender: readBySender && !fromUid.IsZero()}, nil
 }
 
 func enqueueKafkaMessagePersist(ctx context.Context, tenantID types.TenantID, msg *types.Message, attachmentURLs []string, readBySender bool) error {