Refine Kafka persistence topic boundaries

This commit is contained in:
richjohnrambo
2026-09-01 23:23:08 +08:00
parent f8a286cec2
commit 318ce04f7d
13 changed files with 294 additions and 252 deletions
+2 -2
View File
@@ -22,8 +22,8 @@ func TestEnsureIMGroupBusinessTablesUsesSelectedTenantDatabase(t *testing.T) {
if err := ensureIMGroupBusinessTables(exec); err != nil {
t.Fatal(err)
}
if len(exec.statements) != 12 {
t.Fatalf("created %d group tables, want 12", len(exec.statements))
if len(exec.statements) != 13 {
t.Fatalf("created %d group tables, want 13", len(exec.statements))
}
for _, statement := range exec.statements {
if strings.Contains(strings.ToLower(statement), "tinode.") || strings.Contains(strings.ToLower(statement), "create database") {
+4 -68
View File
@@ -2,14 +2,11 @@ package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"strings"
"time"
"github.com/segmentio/kafka-go"
"github.com/tinode/chat/server/logs"
"github.com/tinode/chat/server/store"
"github.com/tinode/chat/server/store/types"
)
@@ -39,12 +36,8 @@ type kafkaFileLifecyclePayload struct {
OccurredAt time.Time `json:"occurred_at"`
}
func startKafkaFileLifecycleConsumer() {
startKafkaConsumer("file_lifecycle", handleKafkaFileLifecycle)
}
func enqueueKafkaFileUploaded(ctx context.Context, tenantID types.TenantID, fd *types.FileDef) error {
if !kafkaEnabled() || fd == nil || fd.TenantID != tenantID || fd.Status != types.UploadCompleted {
if fd == nil || fd.TenantID != tenantID || fd.Status != types.UploadCompleted {
return nil
}
payload := kafkaFileLifecyclePayload{
@@ -63,7 +56,7 @@ func enqueueKafkaFileUploaded(ctx context.Context, tenantID types.TenantID, fd *
}
func enqueueKafkaFileAttachmentsBind(ctx context.Context, tenantID types.TenantID, msg *types.Message, attachmentURLs []string) error {
if !kafkaEnabled() || msg == nil || msg.TenantID != tenantID || len(attachmentURLs) == 0 {
if msg == nil || msg.TenantID != tenantID || len(attachmentURLs) == 0 {
return nil
}
payload := kafkaFileLifecyclePayload{
@@ -80,7 +73,7 @@ func enqueueKafkaFileAttachmentsBind(ctx context.Context, tenantID types.TenantI
}
func enqueueKafkaGroupFileLifecycle(ctx context.Context, tenantID types.TenantID, eventType string, file *types.GroupFile, actor types.Uid) error {
if !kafkaEnabled() || file == nil || file.Topic == "" || file.FileID == "" {
if file == nil || file.Topic == "" || file.FileID == "" {
return nil
}
occurredAt := file.UpdatedAt
@@ -108,64 +101,7 @@ func enqueueKafkaFileLifecycle(ctx context.Context, tenantID types.TenantID, eve
if payload.OccurredAt.IsZero() {
payload.OccurredAt = time.Now().UTC().Round(time.Millisecond)
}
encoded, err := json.Marshal(payload)
if err != nil {
return err
}
eventID := stableIMEventID("file-lifecycle", tenantID, eventType, bizID, payload.SeqID, strings.Join(payload.AttachmentURLs, ","))
err = publishKafkaEnvelope(ctx, "file_lifecycle", bizID, kafkaEnvelope{
EventID: eventID,
TenantID: tenantID,
SourceService: "im-server",
TargetService: "im-server",
EventType: eventType,
BizType: "file",
BizID: bizID,
OccurredAt: payload.OccurredAt,
Payload: encoded,
})
if err != nil {
return err
}
actor := types.ParseUserId(payload.UserID)
if auditErr := enqueueKafkaAuditEvent(context.Background(), tenantID, "user", actor, eventType, "file", bizID, "succeeded", "", map[string]any{
"file_id": payload.FileID,
"file_ids": payload.FileIDs,
"topic": payload.Topic,
"group_file_id": payload.GroupFileID,
"seq_id": payload.SeqID,
}); auditErr != nil {
logs.Warn.Printf("kafka file lifecycle: failed to enqueue audit event type=%s biz=%s: %v", eventType, bizID, auditErr)
}
return nil
}
func handleKafkaFileLifecycle(ctx context.Context, envelope kafkaEnvelope, _ kafka.Message) error {
tenantID, err := kafkaTenantIDFromEnvelope(envelope)
if err != nil {
return err
}
consume, claimed, err := beginKafkaConsumeForTopic(tenantID, envelope, "file_lifecycle")
if err != nil || !claimed {
return err
}
var payload kafkaFileLifecyclePayload
if err = json.Unmarshal(envelope.Payload, &payload); err != nil {
return markKafkaDeadLetter(ctx, tenantID, envelope, consume, "MALFORMED_PAYLOAD", err.Error())
}
if err = executeKafkaFileLifecycle(tenantID, payload); err != nil {
failed, markErr := store.HTTPInfrastructure.MarkKafkaConsumeFailed(tenantID, envelope.EventID, kafkaBus.config.ConsumerGroup,
kafkaCommandErrorReason(err), err.Error(), time.Now().UTC().Round(time.Millisecond))
if markErr != nil {
return markErr
}
if failed != nil && failed.RetryCount >= kafkaBus.config.MaxRetryCount {
return markKafkaDeadLetter(ctx, tenantID, envelope, failed, kafkaCommandErrorReason(err), err.Error())
}
sleepKafkaRetryBackoff(failed.RetryCount)
return err
}
return store.HTTPInfrastructure.MarkKafkaConsumeSucceeded(tenantID, envelope.EventID, kafkaBus.config.ConsumerGroup, time.Now().UTC().Round(time.Millisecond))
return executeKafkaFileLifecycle(tenantID, payload)
}
func executeKafkaFileLifecycle(tenantID types.TenantID, payload kafkaFileLifecyclePayload) error {
+18 -2
View File
@@ -24,6 +24,7 @@ var publishKafkaMessagePersist = publishKafkaEnvelope
func startKafkaMessagePersistConsumer() {
startKafkaConsumer("message_persist", handleKafkaMessagePersist)
startKafkaConsumer("group_message_persist", handleKafkaGroupMessagePersist)
}
func saveMessageMaybeKafka(ctx context.Context, tenantID types.TenantID, msg *types.Message, attachmentURLs []string, readBySender bool) (*types.MessageSaveResult, error) {
@@ -79,10 +80,18 @@ func enqueueKafkaMessagePersist(ctx context.Context, tenantID types.TenantID, ms
Actor: &kafkaActor{Type: "user", ID: types.ParseUid(msg.From).UserId()},
Payload: payload,
}
return publishKafkaMessagePersist(ctx, "message_persist", msg.Topic, envelope)
return publishKafkaMessagePersist(ctx, kafkaMessagePersistTopicName(msg.Topic), msg.Topic, envelope)
}
func handleKafkaMessagePersist(ctx context.Context, envelope kafkaEnvelope, _ kafka.Message) error {
return handleKafkaMessagePersistFromTopic(ctx, envelope, "message_persist")
}
func handleKafkaGroupMessagePersist(ctx context.Context, envelope kafkaEnvelope, _ kafka.Message) error {
return handleKafkaMessagePersistFromTopic(ctx, envelope, "group_message_persist")
}
func handleKafkaMessagePersistFromTopic(ctx context.Context, envelope kafkaEnvelope, sourceTopicName string) error {
tenantID := envelope.TenantID
var err error
if tenantID.IsZero() {
@@ -91,7 +100,7 @@ func handleKafkaMessagePersist(ctx context.Context, envelope kafkaEnvelope, _ ka
return err
}
}
consume, claimed, err := beginKafkaConsumeForTopic(tenantID, envelope, "message_persist")
consume, claimed, err := beginKafkaConsumeForTopic(tenantID, envelope, sourceTopicName)
if err != nil || !claimed {
return err
}
@@ -137,6 +146,13 @@ func handleKafkaMessagePersist(ctx context.Context, envelope kafkaEnvelope, _ ka
return store.HTTPInfrastructure.MarkKafkaConsumeSucceeded(tenantID, envelope.EventID, kafkaBus.config.ConsumerGroup, time.Now().UTC().Round(time.Millisecond))
}
func kafkaMessagePersistTopicName(topic string) string {
if types.GetTopicCat(topic) == types.TopicCatGrp {
return "group_message_persist"
}
return "message_persist"
}
func validateKafkaMessageKind(payload kafkaMessagePersistPayload) error {
if payload.Message.Topic == "" || payload.Message.SeqId <= 0 {
return types.ErrMalformed
+22 -24
View File
@@ -17,18 +17,17 @@ import (
)
const (
kafkaTopicAdminCommand = "im.admin.command"
kafkaTopicAdminCommandResult = "im.admin.command.result"
kafkaTopicConfigEvent = "im.config.event"
kafkaTopicBusinessEvent = "im.business.event"
kafkaTopicMessagePersist = "im.message.persist"
kafkaTopicMessagePush = "im.message.push"
kafkaTopicFileSecurity = "im.file.security"
kafkaTopicFileLifecycle = "im.file.lifecycle"
kafkaTopicWorkbenchEvent = "im.workbench.event"
kafkaTopicCacheInvalidate = "im.cache.invalidate"
kafkaTopicAuditEvent = "im.audit.event"
kafkaTopicDeadLetter = "im.dead.letter"
kafkaTopicAdminCommand = "im.admin.command"
kafkaTopicAdminCommandResult = "im.admin.command.result"
kafkaTopicConfigEvent = "im.config.event"
kafkaTopicBusinessEvent = "im.business.event"
kafkaTopicMessagePersist = "im.message.persist"
kafkaTopicGroupMessagePersist = "im.group.message.persist"
kafkaTopicMessagePush = "im.message.push"
kafkaTopicFileSecurity = "im.file.security"
kafkaTopicCacheInvalidate = "im.cache.invalidate"
kafkaTopicAuditEvent = "im.audit.event"
kafkaTopicDeadLetter = "im.dead.letter"
)
type kafkaConfig struct {
@@ -199,18 +198,17 @@ func kafkaSASLMechanism(cfg kafkaConfig) (sasl.Mechanism, error) {
func kafkaTopics(overrides map[string]string) map[string]string {
topics := map[string]string{
"admin_command": kafkaTopicAdminCommand,
"admin_command_result": kafkaTopicAdminCommandResult,
"config_event": kafkaTopicConfigEvent,
"business_event": kafkaTopicBusinessEvent,
"message_persist": kafkaTopicMessagePersist,
"message_push": kafkaTopicMessagePush,
"file_security": kafkaTopicFileSecurity,
"file_lifecycle": kafkaTopicFileLifecycle,
"workbench_event": kafkaTopicWorkbenchEvent,
"cache_invalidate": kafkaTopicCacheInvalidate,
"audit_event": kafkaTopicAuditEvent,
"dead_letter": kafkaTopicDeadLetter,
"admin_command": kafkaTopicAdminCommand,
"admin_command_result": kafkaTopicAdminCommandResult,
"config_event": kafkaTopicConfigEvent,
"business_event": kafkaTopicBusinessEvent,
"message_persist": kafkaTopicMessagePersist,
"group_message_persist": kafkaTopicGroupMessagePersist,
"message_push": kafkaTopicMessagePush,
"file_security": kafkaTopicFileSecurity,
"cache_invalidate": kafkaTopicCacheInvalidate,
"audit_event": kafkaTopicAuditEvent,
"dead_letter": kafkaTopicDeadLetter,
}
for key, value := range overrides {
if value != "" {
+17 -50
View File
@@ -5,8 +5,6 @@ import (
"encoding/json"
"time"
"github.com/segmentio/kafka-go"
"github.com/tinode/chat/server/store"
"github.com/tinode/chat/server/store/types"
)
@@ -34,15 +32,10 @@ type kafkaAuditPayload struct {
OccurredAt time.Time `json:"occurred_at"`
}
func startKafkaWorkbenchEventConsumer() {
startKafkaConsumer("workbench_event", handleKafkaWorkbenchEvent)
}
var publishKafkaAuditEventEnvelope = publishKafkaEnvelope
func enqueueKafkaWorkbenchEvent(ctx context.Context, tenantID types.TenantID, actor types.Uid, module, action, resourceID string,
targets []types.Uid, title, summary string, data map[string]any) error {
if !kafkaEnabled() {
return nil
}
targetUserIDs := make([]string, 0, len(targets))
for _, target := range targets {
if !target.IsZero() {
@@ -60,7 +53,14 @@ func enqueueKafkaWorkbenchEvent(ctx context.Context, tenantID types.TenantID, ac
Data: data,
OccurredAt: time.Now().UTC().Round(time.Millisecond),
}
return enqueueKafkaTypedEvent(ctx, tenantID, "workbench_event", "workbench."+action, "workbench", resourceID, actor, payload)
envelope := kafkaEnvelope{
EventID: stableIMEventID("workbench-sync", tenantID, "workbench."+action, resourceID, payload.OccurredAt),
TenantID: tenantID,
EventType: "workbench." + action,
BizType: "workbench",
BizID: resourceID,
}
return executeKafkaWorkbenchEvent(tenantID, envelope, payload)
}
func enqueueKafkaAuditEvent(ctx context.Context, tenantID types.TenantID, actorType string, actor types.Uid,
@@ -83,58 +83,25 @@ func enqueueKafkaAuditEvent(ctx context.Context, tenantID types.TenantID, actorT
Data: data,
OccurredAt: time.Now().UTC().Round(time.Millisecond),
}
return enqueueKafkaTypedEvent(ctx, tenantID, "audit_event", "audit."+action, "audit", resource+":"+resourceID, actor, payload)
}
func enqueueKafkaTypedEvent(ctx context.Context, tenantID types.TenantID, topicName, eventType, bizType, bizID string, actor types.Uid, payload any) error {
encoded, err := json.Marshal(payload)
if err != nil {
return err
}
eventID := stableIMEventID(topicName, tenantID, eventType, bizID, encoded)
envelope := kafkaEnvelope{
EventID: eventID,
EventID: stableIMEventID("audit_event", tenantID, "audit."+action, resource+":"+resourceID, encoded),
TenantID: tenantID,
SourceService: "im-server",
TargetService: "im-server",
EventType: eventType,
BizType: bizType,
BizID: bizID,
OccurredAt: time.Now().UTC(),
TargetService: "hotgo-admin",
EventType: "audit." + action,
BizType: "audit",
BizID: resource + ":" + resourceID,
OccurredAt: payload.OccurredAt,
Payload: encoded,
}
if !actor.IsZero() {
envelope.Actor = &kafkaActor{Type: "user", ID: actor.UserId()}
envelope.Actor = &kafkaActor{Type: actorType, ID: actorID}
}
return publishKafkaEnvelope(ctx, topicName, bizID, envelope)
}
func handleKafkaWorkbenchEvent(ctx context.Context, envelope kafkaEnvelope, _ kafka.Message) error {
tenantID, err := kafkaTenantIDFromEnvelope(envelope)
if err != nil {
return err
}
consume, claimed, err := beginKafkaConsumeForTopic(tenantID, envelope, "workbench_event")
if err != nil || !claimed {
return err
}
var payload kafkaWorkbenchPayload
if err = json.Unmarshal(envelope.Payload, &payload); err != nil {
return markKafkaDeadLetter(ctx, tenantID, envelope, consume, "MALFORMED_PAYLOAD", err.Error())
}
if err = executeKafkaWorkbenchEvent(tenantID, envelope, payload); err != nil {
failed, markErr := store.HTTPInfrastructure.MarkKafkaConsumeFailed(tenantID, envelope.EventID, kafkaBus.config.ConsumerGroup,
kafkaCommandErrorReason(err), err.Error(), time.Now().UTC().Round(time.Millisecond))
if markErr != nil {
return markErr
}
if failed != nil && failed.RetryCount >= kafkaBus.config.MaxRetryCount {
return markKafkaDeadLetter(ctx, tenantID, envelope, failed, kafkaCommandErrorReason(err), err.Error())
}
sleepKafkaRetryBackoff(failed.RetryCount)
return err
}
return store.HTTPInfrastructure.MarkKafkaConsumeSucceeded(tenantID, envelope.EventID, kafkaBus.config.ConsumerGroup, time.Now().UTC().Round(time.Millisecond))
return publishKafkaAuditEventEnvelope(ctx, "audit_event", envelope.BizID, envelope)
}
func executeKafkaWorkbenchEvent(tenantID types.TenantID, envelope kafkaEnvelope, payload kafkaWorkbenchPayload) error {
+139
View File
@@ -0,0 +1,139 @@
package main
import (
"context"
"encoding/json"
"testing"
"time"
"github.com/golang/mock/gomock"
"github.com/tinode/chat/server/store"
"github.com/tinode/chat/server/store/mock_store"
"github.com/tinode/chat/server/store/types"
)
func TestKafkaTopicsKeepOnlyMessageInternalAsyncTopics(t *testing.T) {
topics := kafkaTopics(nil)
if topics["message_persist"] != "im.message.persist" {
t.Fatalf("unexpected private message persist topic: %q", topics["message_persist"])
}
if topics["group_message_persist"] != "im.group.message.persist" {
t.Fatalf("unexpected group message persist topic: %q", topics["group_message_persist"])
}
if topics["audit_event"] != "im.audit.event" {
t.Fatalf("unexpected audit event topic: %q", topics["audit_event"])
}
for _, removed := range []string{"file_lifecycle", "workbench_event"} {
if _, ok := topics[removed]; ok {
t.Fatalf("internal sync topic %q must not be configured", removed)
}
}
}
func TestEnqueueKafkaAuditEventPublishesCrossSystemEvent(t *testing.T) {
oldKafkaBus := kafkaBus
oldPublisher := publishKafkaAuditEventEnvelope
defer func() {
kafkaBus = oldKafkaBus
publishKafkaAuditEventEnvelope = oldPublisher
}()
kafkaBus = &kafkaRuntime{}
var publishedTopic string
var publishedKey string
var publishedEnvelope kafkaEnvelope
publishKafkaAuditEventEnvelope = func(_ context.Context, topicName, key string, envelope kafkaEnvelope) error {
publishedTopic = topicName
publishedKey = key
publishedEnvelope = envelope
return nil
}
err := enqueueKafkaAuditEvent(context.Background(), 7, "admin", types.Uid(42),
"group.update", "group", "grp-test", "succeeded", "", map[string]any{"topic": "grp-test"})
if err != nil {
t.Fatalf("enqueueKafkaAuditEvent returned error: %v", err)
}
if publishedTopic != "audit_event" {
t.Fatalf("unexpected audit publish topic: %q", publishedTopic)
}
if publishedKey != "group:grp-test" {
t.Fatalf("unexpected audit publish key: %q", publishedKey)
}
if publishedEnvelope.TargetService != "hotgo-admin" || publishedEnvelope.EventType != "audit.group.update" {
t.Fatalf("unexpected audit envelope: %#v", publishedEnvelope)
}
if publishedEnvelope.Actor == nil || publishedEnvelope.Actor.Type != "admin" || publishedEnvelope.Actor.ID != types.Uid(42).UserId() {
t.Fatalf("unexpected audit actor: %#v", publishedEnvelope.Actor)
}
var payload kafkaAuditPayload
if err = json.Unmarshal(publishedEnvelope.Payload, &payload); err != nil {
t.Fatalf("invalid audit payload: %v", err)
}
if payload.Resource != "group" || payload.ResourceID != "grp-test" || payload.Result != "succeeded" {
t.Fatalf("unexpected audit payload: %#v", payload)
}
}
func TestEnqueueKafkaFileAttachmentsBindRunsSynchronously(t *testing.T) {
ctrl := gomock.NewController(t)
defer ctrl.Finish()
oldKafkaBus := kafkaBus
oldFiles := store.Files
defer func() {
kafkaBus = oldKafkaBus
store.Files = oldFiles
}()
kafkaBus = &kafkaRuntime{}
files := mock_store.NewMockFilePersistenceInterface(ctrl)
store.Files = files
msgID := types.Uid(101)
msg := &types.Message{
ObjHeader: types.ObjHeader{Id: msgID.String(), CreatedAt: time.Date(2026, 9, 1, 10, 0, 0, 0, time.UTC)},
TenantID: 7,
Topic: "usr42",
SeqId: 3,
From: types.Uid(42).String(),
Head: types.KVMap{"x-im-type": "image"},
}
attachments := []string{"https://api.118im.com/v1/im/files/file_001"}
files.EXPECT().LinkAttachments(types.TenantID(7), "", msgID, attachments).Return(nil)
if err := enqueueKafkaFileAttachmentsBind(context.Background(), 7, msg, attachments); err != nil {
t.Fatalf("enqueueKafkaFileAttachmentsBind returned error: %v", err)
}
}
func TestEnqueueKafkaWorkbenchEventRunsSynchronously(t *testing.T) {
oldKafkaBus := kafkaBus
oldInfrastructure := store.HTTPInfrastructure
oldSecurity := store.Security
defer func() {
kafkaBus = oldKafkaBus
store.HTTPInfrastructure = oldInfrastructure
store.Security = oldSecurity
}()
kafkaBus = &kafkaRuntime{}
infrastructure := &testHTTPInfrastructureStore{}
security := &testSecurityStore{}
store.HTTPInfrastructure = infrastructure
store.Security = security
err := enqueueKafkaWorkbenchEvent(context.Background(), 7, types.Uid(42), "meeting", "meeting.created", "meeting-1",
[]types.Uid{42, 43}, "会议通知", "周会", map[string]any{"meeting_id": "meeting-1"})
if err != nil {
t.Fatalf("enqueueKafkaWorkbenchEvent returned error: %v", err)
}
if len(security.notifications) != 1 {
t.Fatalf("expected one notification for non-actor target, got %#v", security.notifications)
}
if security.notifications[0].UserID != types.Uid(43) || security.notifications[0].EventType != "meeting.created" {
t.Fatalf("unexpected notification: %#v", security.notifications[0])
}
if len(infrastructure.events) != 2 {
t.Fatalf("expected business events for both targets, got %#v", infrastructure.events)
}
}
-2
View File
@@ -696,8 +696,6 @@ func main() {
startKafkaConfigEventConsumer()
startKafkaCacheInvalidateConsumer()
startKafkaMessagePersistConsumer()
startKafkaFileLifecycleConsumer()
startKafkaWorkbenchEventConsumer()
startKafkaDeadLetterConsumer()
stopExternalBusinessEvents := startIMExternalBusinessEventPoller()
defer stopExternalBusinessEvents()
+1 -2
View File
@@ -130,10 +130,9 @@
"config_event": "im.config.event",
"business_event": "im.business.event",
"message_persist": "im.message.persist",
"group_message_persist": "im.group.message.persist",
"message_push": "im.message.push",
"file_security": "im.file.security",
"file_lifecycle": "im.file.lifecycle",
"workbench_event": "im.workbench.event",
"cache_invalidate": "im.cache.invalidate",
"audit_event": "im.audit.event",
"dead_letter": "im.dead.letter"
+1 -2
View File
@@ -130,10 +130,9 @@
"config_event": "im.config.event",
"business_event": "im.business.event",
"message_persist": "im.message.persist",
"group_message_persist": "im.group.message.persist",
"message_push": "im.message.push",
"file_security": "im.file.security",
"file_lifecycle": "im.file.lifecycle",
"workbench_event": "im.workbench.event",
"cache_invalidate": "im.cache.invalidate",
"audit_event": "im.audit.event",
"dead_letter": "im.dead.letter"
+1 -1
View File
@@ -948,7 +948,7 @@ func TestSaveMessageMaybeKafkaPublishesWithoutDBPersist(t *testing.T) {
if msg.ClientMsgID != "client-1" {
t.Fatalf("client message id not normalized: %q", msg.ClientMsgID)
}
if publishedTopic != "message_persist" {
if publishedTopic != "group_message_persist" {
t.Fatalf("unexpected kafka topic: %q", publishedTopic)
}
if publishedKey != "grp-test" {