package main import ( "context" "encoding/json" "errors" "fmt" "time" "github.com/segmentio/kafka-go" "github.com/tinode/chat/server/logs" "github.com/tinode/chat/server/store" "github.com/tinode/chat/server/store/types" ) type kafkaMessagePersistPayload struct { Message types.Message `json:"message"` AttachmentURLs []string `json:"attachment_urls,omitempty"` ReadBySender bool `json:"read_by_sender,omitempty"` MessageKind string `json:"message_kind"` } func startKafkaMessagePersistConsumer() { startKafkaConsumer("message_persist", handleKafkaMessagePersist) } func saveMessageMaybeKafka(ctx context.Context, tenantID types.TenantID, msg *types.Message, attachmentURLs []string, readBySender bool) (*types.MessageSaveResult, error) { if !kafkaEnabled() { return store.Messages.Save(tenantID, msg, attachmentURLs, readBySender) } saveResult, err := store.PrepareMessagePersist(tenantID, msg, attachmentURLs, readBySender) if err != nil || saveResult == nil || saveResult.Duplicate { return saveResult, err } if readBySender && !types.ParseUid(msg.From).IsZero() { saveResult.MarkedReadBySender = true } if err = enqueueKafkaMessagePersist(ctx, tenantID, msg, attachmentURLs, readBySender); err != nil { return nil, err } return saveResult, nil } func enqueueKafkaMessagePersist(_ context.Context, tenantID types.TenantID, msg *types.Message, attachmentURLs []string, readBySender bool) error { tenantCode := "" if store.Tenants != nil { if tenant, err := store.Tenants.GetByID(tenantID); err == nil && tenant != nil { tenantCode = tenant.Code } else { logs.Warn.Printf("kafka message persist: failed to resolve tenant code for tenant %d: %v", tenantID, err) } } kind := "p2p" if types.GetTopicCat(msg.Topic) == types.TopicCatGrp { kind = "group" } payload, err := json.Marshal(kafkaMessagePersistPayload{ Message: *msg, AttachmentURLs: attachmentURLs, ReadBySender: readBySender, MessageKind: kind, }) if err != nil { return err } eventID := fmt.Sprintf("msg.persist.%d.%s.%d", tenantID, msg.Topic, msg.SeqId) envelope := kafkaEnvelope{ EventID: eventID, TenantCode: tenantCode, SourceService: "im-server", TargetService: "im-server", EventType: "message.persist." + kind, BizType: "message", BizID: fmt.Sprintf("%s:%d", msg.Topic, msg.SeqId), IdempotencyKey: msg.ClientMsgID, Version: msg.SeqId, OccurredAt: msg.CreatedAt, Actor: &kafkaActor{Type: "user", ID: types.ParseUid(msg.From).UserId()}, Payload: payload, } return enqueueKafkaOutbox(tenantID, "message_persist", msg.Topic, envelope) } func handleKafkaMessagePersist(ctx context.Context, envelope kafkaEnvelope, _ kafka.Message) error { tenantID, err := kafkaTenantID(envelope.TenantCode) if err != nil { return err } consume, claimed, err := beginKafkaConsumeForTopic(tenantID, envelope, "message_persist") if err != nil || !claimed { return err } var payload kafkaMessagePersistPayload if err = json.Unmarshal(envelope.Payload, &payload); err != nil { return markKafkaDeadLetter(ctx, tenantID, envelope, consume, "MALFORMED_PAYLOAD", err.Error()) } if payload.Message.TenantID == types.ZeroTenantID { payload.Message.TenantID = tenantID } if payload.Message.TenantID != tenantID { return markKafkaDeadLetter(ctx, tenantID, envelope, consume, "TENANT_MISMATCH", "message tenant does not match envelope tenant") } if err = validateKafkaMessageKind(payload); err != nil { return markKafkaDeadLetter(ctx, tenantID, envelope, consume, "MALFORMED_PAYLOAD", err.Error()) } if _, err = store.PersistPreparedMessage(tenantID, &payload.Message, nil, payload.ReadBySender); err != nil && !errors.Is(err, types.ErrDuplicate) { failed, markErr := store.HTTPInfrastructure.MarkKafkaConsumeFailed(tenantID, envelope.EventID, kafkaBus.config.ConsumerGroup, kafkaCommandErrorReason(err), err.Error(), time.Now().UTC().Round(time.Millisecond)) if markErr != nil { return markErr } if failed != nil && failed.RetryCount >= kafkaBus.config.MaxRetryCount { return markKafkaDeadLetter(ctx, tenantID, envelope, failed, kafkaCommandErrorReason(err), err.Error()) } sleepKafkaRetryBackoff(failed.RetryCount) return err } if len(payload.AttachmentURLs) > 0 { if err = enqueueKafkaFileAttachmentsBind(ctx, tenantID, &payload.Message, payload.AttachmentURLs); err != nil { failed, markErr := store.HTTPInfrastructure.MarkKafkaConsumeFailed(tenantID, envelope.EventID, kafkaBus.config.ConsumerGroup, "KAFKA_FILE_LIFECYCLE_FAILED", err.Error(), time.Now().UTC().Round(time.Millisecond)) if markErr != nil { return markErr } if failed != nil && failed.RetryCount >= kafkaBus.config.MaxRetryCount { return markKafkaDeadLetter(ctx, tenantID, envelope, failed, "KAFKA_FILE_LIFECYCLE_FAILED", err.Error()) } sleepKafkaRetryBackoff(failed.RetryCount) return err } } return store.HTTPInfrastructure.MarkKafkaConsumeSucceeded(tenantID, envelope.EventID, kafkaBus.config.ConsumerGroup, time.Now().UTC().Round(time.Millisecond)) } func validateKafkaMessageKind(payload kafkaMessagePersistPayload) error { if payload.Message.Topic == "" || payload.Message.SeqId <= 0 { return types.ErrMalformed } isGroup := types.GetTopicCat(payload.Message.Topic) == types.TopicCatGrp switch payload.MessageKind { case "group": if !isGroup { return types.ErrMalformed } case "p2p", "": if isGroup { return types.ErrMalformed } default: return types.ErrMalformed } return nil }