227 lines
8.5 KiB
Go
227 lines
8.5 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"time"
|
|
|
|
"github.com/segmentio/kafka-go"
|
|
"github.com/tinode/chat/server/logs"
|
|
"github.com/tinode/chat/server/store"
|
|
"github.com/tinode/chat/server/store/types"
|
|
)
|
|
|
|
type kafkaMessagePersistPayload struct {
|
|
Message types.Message `json:"message"`
|
|
AttachmentURLs []string `json:"attachment_urls,omitempty"`
|
|
ReadBySender bool `json:"read_by_sender,omitempty"`
|
|
MessageKind string `json:"message_kind"`
|
|
ShardMonth string `json:"shard_month,omitempty"`
|
|
}
|
|
|
|
var publishKafkaMessagePersist = publishKafkaEnvelope
|
|
|
|
func startKafkaMessagePersistConsumer() {
|
|
startKafkaConsumer("message_persist", handleKafkaMessagePersist)
|
|
startKafkaConsumer("group_message_persist", handleKafkaGroupMessagePersist)
|
|
}
|
|
|
|
func saveMessageMaybeKafka(ctx context.Context, tenantID types.TenantID, msg *types.Message, attachmentURLs []string, readBySender bool) (*types.MessageSaveResult, error) {
|
|
if !kafkaEnabled() {
|
|
return store.Messages.Save(tenantID, msg, attachmentURLs, readBySender)
|
|
}
|
|
if msg == nil || msg.TenantID != tenantID || msg.Topic == "" || msg.SeqId <= 0 {
|
|
return nil, types.ErrMalformed
|
|
}
|
|
msg.InitTimes()
|
|
if msg.Uid().IsZero() {
|
|
msg.SetUid(store.Store.GetUid())
|
|
}
|
|
isGroupMessage := types.GetTopicCat(msg.Topic) == types.TopicCatGrp
|
|
clientMsgID, err := store.NormalizeClientMsgID(msg.Head)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
msg.ClientMsgID = clientMsgID
|
|
fromUid := types.ParseUid(msg.From)
|
|
if clientMsgID != "" {
|
|
claim, claimed, runtimeReady, claimErr := store.ClaimMessageUnique(tenantID, msg.Topic, fromUid, clientMsgID)
|
|
if claimErr != nil {
|
|
return nil, claimErr
|
|
}
|
|
if runtimeReady && !claimed {
|
|
if claim != nil && claim.Seq > 0 {
|
|
msg.SeqId = claim.Seq
|
|
return &types.MessageSaveResult{SeqId: claim.Seq, Duplicate: true, Pending: claim.Status == "pending"}, nil
|
|
}
|
|
if claim != nil && claim.Status == "pending" {
|
|
return &types.MessageSaveResult{Duplicate: true, Pending: true}, types.ErrDuplicate
|
|
}
|
|
return nil, types.ErrDuplicate
|
|
}
|
|
}
|
|
candidateMonth := msg.CreatedAt.UTC().Format("200601")
|
|
if seq, shardMonth, runtimeReady, seqErr := store.ReserveMessageSeq(tenantID, msg.Topic, msg.SeqId-1, candidateMonth); seqErr != nil {
|
|
if clientMsgID != "" {
|
|
_ = store.MarkMessageUniqueFailed(tenantID, msg.Topic, fromUid, clientMsgID)
|
|
}
|
|
return nil, seqErr
|
|
} else if runtimeReady {
|
|
msg.SeqId = seq
|
|
if isGroupMessage {
|
|
msg.ShardMonth = shardMonth
|
|
}
|
|
}
|
|
if msg.ShardMonth == "" && isGroupMessage {
|
|
msg.ShardMonth = candidateMonth
|
|
}
|
|
if err = enqueueKafkaMessagePersist(ctx, tenantID, msg, attachmentURLs, readBySender); err != nil {
|
|
if clientMsgID != "" {
|
|
if markErr := store.MarkMessageUniquePending(tenantID, msg.Topic, fromUid, clientMsgID, msg.SeqId); markErr != nil {
|
|
logs.Warn.Printf("kafka message persist: failed to mark message unique key pending topic=%s seq=%d: %v", msg.Topic, msg.SeqId, markErr)
|
|
}
|
|
}
|
|
logs.Warn.Printf("kafka message persist: publish state unknown topic=%s seq=%d client_mid=%s err=%v", msg.Topic, msg.SeqId, clientMsgID, err)
|
|
return &types.MessageSaveResult{SeqId: msg.SeqId, Pending: true, MarkedReadBySender: readBySender && !fromUid.IsZero()}, err
|
|
}
|
|
if clientMsgID != "" {
|
|
if err = store.MarkMessageUniquePending(tenantID, msg.Topic, fromUid, clientMsgID, msg.SeqId); err != nil {
|
|
logs.Warn.Printf("kafka message persist: failed to mark message unique key pending topic=%s seq=%d: %v", msg.Topic, msg.SeqId, err)
|
|
}
|
|
}
|
|
if isGroupMessage {
|
|
if cacheErr := store.CachePendingHotMessage(tenantID, msg); cacheErr != nil {
|
|
logs.Warn.Printf("kafka message persist: failed to cache pending hot message after publish topic=%s seq=%d: %v", msg.Topic, msg.SeqId, cacheErr)
|
|
}
|
|
}
|
|
return &types.MessageSaveResult{SeqId: msg.SeqId, Pending: true, MarkedReadBySender: readBySender && !fromUid.IsZero()}, nil
|
|
}
|
|
|
|
func enqueueKafkaMessagePersist(ctx context.Context, tenantID types.TenantID, msg *types.Message, attachmentURLs []string, readBySender bool) error {
|
|
kind := "p2p"
|
|
if types.GetTopicCat(msg.Topic) == types.TopicCatGrp {
|
|
kind = "group"
|
|
}
|
|
payload, err := json.Marshal(kafkaMessagePersistPayload{
|
|
Message: *msg,
|
|
AttachmentURLs: attachmentURLs,
|
|
ReadBySender: readBySender,
|
|
MessageKind: kind,
|
|
ShardMonth: msg.ShardMonth,
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
eventID := fmt.Sprintf("msg.persist.%d.%s.%d", tenantID, msg.Topic, msg.SeqId)
|
|
envelope := kafkaEnvelope{
|
|
EventID: eventID,
|
|
TenantID: tenantID,
|
|
SourceService: "im-server",
|
|
TargetService: "im-server",
|
|
EventType: "message.persist." + kind,
|
|
BizType: "message",
|
|
BizID: fmt.Sprintf("%s:%d", msg.Topic, msg.SeqId),
|
|
IdempotencyKey: msg.ClientMsgID,
|
|
Version: msg.SeqId,
|
|
OccurredAt: msg.CreatedAt,
|
|
Actor: &kafkaActor{Type: "user", ID: types.ParseUid(msg.From).UserId()},
|
|
Payload: payload,
|
|
}
|
|
return publishKafkaMessagePersist(ctx, kafkaMessagePersistTopicName(msg.Topic), msg.Topic, envelope)
|
|
}
|
|
|
|
func handleKafkaMessagePersist(ctx context.Context, envelope kafkaEnvelope, _ kafka.Message) error {
|
|
return handleKafkaMessagePersistFromTopic(ctx, envelope, "message_persist")
|
|
}
|
|
|
|
func handleKafkaGroupMessagePersist(ctx context.Context, envelope kafkaEnvelope, _ kafka.Message) error {
|
|
return handleKafkaMessagePersistFromTopic(ctx, envelope, "group_message_persist")
|
|
}
|
|
|
|
func handleKafkaMessagePersistFromTopic(ctx context.Context, envelope kafkaEnvelope, sourceTopicName string) error {
|
|
tenantID := envelope.TenantID
|
|
var err error
|
|
if tenantID.IsZero() {
|
|
tenantID, err = kafkaTenantID(envelope.TenantCode)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
}
|
|
consume, claimed, err := beginKafkaConsumeForTopic(tenantID, envelope, sourceTopicName)
|
|
if err != nil || !claimed {
|
|
return err
|
|
}
|
|
var payload kafkaMessagePersistPayload
|
|
if err = json.Unmarshal(envelope.Payload, &payload); err != nil {
|
|
return markKafkaDeadLetter(ctx, tenantID, envelope, consume, "MALFORMED_PAYLOAD", err.Error())
|
|
}
|
|
if payload.Message.TenantID == types.ZeroTenantID {
|
|
payload.Message.TenantID = tenantID
|
|
}
|
|
if payload.Message.ShardMonth == "" {
|
|
payload.Message.ShardMonth = payload.ShardMonth
|
|
}
|
|
if payload.Message.TenantID != tenantID {
|
|
return markKafkaDeadLetter(ctx, tenantID, envelope, consume, "TENANT_MISMATCH", "message tenant does not match envelope tenant")
|
|
}
|
|
if err = validateKafkaMessageKind(payload); err != nil {
|
|
return markKafkaDeadLetter(ctx, tenantID, envelope, consume, "MALFORMED_PAYLOAD", err.Error())
|
|
}
|
|
if _, err = store.PersistPreparedMessage(tenantID, &payload.Message, nil, payload.ReadBySender); err != nil && !errors.Is(err, types.ErrDuplicate) {
|
|
failed, markErr := store.HTTPInfrastructure.MarkKafkaConsumeFailed(tenantID, envelope.EventID, kafkaBus.config.ConsumerGroup,
|
|
kafkaCommandErrorReason(err), err.Error(), time.Now().UTC().Round(time.Millisecond))
|
|
if markErr != nil {
|
|
return markErr
|
|
}
|
|
if failed != nil && failed.RetryCount >= kafkaBus.config.MaxRetryCount {
|
|
return markKafkaDeadLetter(ctx, tenantID, envelope, failed, kafkaCommandErrorReason(err), err.Error())
|
|
}
|
|
sleepKafkaRetryBackoff(failed.RetryCount)
|
|
return err
|
|
}
|
|
if len(payload.AttachmentURLs) > 0 {
|
|
if err = enqueueKafkaFileAttachmentsBind(ctx, tenantID, &payload.Message, payload.AttachmentURLs); err != nil {
|
|
failed, markErr := store.HTTPInfrastructure.MarkKafkaConsumeFailed(tenantID, envelope.EventID, kafkaBus.config.ConsumerGroup,
|
|
"KAFKA_FILE_LIFECYCLE_FAILED", err.Error(), time.Now().UTC().Round(time.Millisecond))
|
|
if markErr != nil {
|
|
return markErr
|
|
}
|
|
if failed != nil && failed.RetryCount >= kafkaBus.config.MaxRetryCount {
|
|
return markKafkaDeadLetter(ctx, tenantID, envelope, failed, "KAFKA_FILE_LIFECYCLE_FAILED", err.Error())
|
|
}
|
|
sleepKafkaRetryBackoff(failed.RetryCount)
|
|
return err
|
|
}
|
|
}
|
|
return store.HTTPInfrastructure.MarkKafkaConsumeSucceeded(tenantID, envelope.EventID, kafkaBus.config.ConsumerGroup, time.Now().UTC().Round(time.Millisecond))
|
|
}
|
|
|
|
func kafkaMessagePersistTopicName(topic string) string {
|
|
if types.GetTopicCat(topic) == types.TopicCatGrp {
|
|
return "group_message_persist"
|
|
}
|
|
return "message_persist"
|
|
}
|
|
|
|
func validateKafkaMessageKind(payload kafkaMessagePersistPayload) error {
|
|
if payload.Message.Topic == "" || payload.Message.SeqId <= 0 {
|
|
return types.ErrMalformed
|
|
}
|
|
isGroup := types.GetTopicCat(payload.Message.Topic) == types.TopicCatGrp
|
|
switch payload.MessageKind {
|
|
case "group":
|
|
if !isGroup {
|
|
return types.ErrMalformed
|
|
}
|
|
case "p2p", "":
|
|
if isGroup {
|
|
return types.ErrMalformed
|
|
}
|
|
default:
|
|
return types.ErrMalformed
|
|
}
|
|
return nil
|
|
}
|