pro/im_server
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: harden multi-pod delivery and load verification

Browse Source
This commit is contained in:
richjohnrambo
2026-09-27 03:37:32 +08:00
parent a2de100f2b
commit ce3250be64
33 changed files with 1340 additions and 94 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/kafka_message_persist.go
+27 -2
View File
@@ -10,6 +10,7 @@ import (
"github.com/segmentio/kafka-go"
"github.com/tinode/chat/server/logs"
immetrics "github.com/tinode/chat/server/metrics"
"github.com/tinode/chat/server/store"
"github.com/tinode/chat/server/store/types"
)
@@ -160,11 +161,18 @@ func handleKafkaMessagePersistBatch(ctx context.Context, messages []kafka.Messag
groups[key] = append(groups[key], item)
}
for _, items := range groups {
kind := items[0].payload.MessageKind
if kind == "" {
kind = "p2p"
}
started := time.Now()
batch := make([]store.PreparedMessageBatchItem, 0, len(items))
for _, item := range items {
batch = append(batch, store.PreparedMessageBatchItem{Message: &item.payload.Message, ReadBySender: item.payload.ReadBySender})
}
if err := store.PersistPreparedMessageBatch(items[0].tenantID, batch); err == nil {
immetrics.KafkaMessagePersistTotal.WithLabelValues(kind, "batch_ok").Add(float64(len(items)))
immetrics.MessagePersistDuration.WithLabelValues(kind, "batch_ok").Observe(time.Since(started).Seconds())
for _, item := range items {
if err = finishKafkaMessagePersist(ctx, item); err != nil {
if err = persistKafkaMessageIndividually(ctx, item); err != nil {
@@ -174,6 +182,8 @@ func handleKafkaMessagePersistBatch(ctx context.Context, messages []kafka.Messag
}
continue
} else {
immetrics.KafkaMessagePersistTotal.WithLabelValues(kind, "batch_failed").Add(float64(len(items)))
immetrics.MessagePersistDuration.WithLabelValues(kind, "batch_failed").Observe(time.Since(started).Seconds())
logs.Warn.Printf("kafka message batch persist failed, degrading to singles: tenant=%d kind=%s shard=%s size=%d err=%v",
items[0].tenantID, items[0].payload.MessageKind, items[0].payload.Message.ShardMonth, len(items), err)
}
@@ -227,6 +237,11 @@ func prepareKafkaMessagePersistItem(ctx context.Context, message kafka.Message,
}
func persistKafkaMessageIndividually(ctx context.Context, item *kafkaMessagePersistItem) error {
kind := item.payload.MessageKind
if kind == "" {
kind = "p2p"
}
started := time.Now()
operation := func() error {
err := store.PersistPreparedMessageBatch(item.tenantID, []store.PreparedMessageBatchItem{{
Message: &item.payload.Message, ReadBySender: item.payload.ReadBySender,
@@ -237,14 +252,24 @@ func persistKafkaMessageIndividually(ctx context.Context, item *kafkaMessagePers
if err != nil && !errors.Is(err, types.ErrDuplicate) {
return err
}
return finishKafkaMessagePersist(ctx, item)
if err = finishKafkaMessagePersist(ctx, item); err != nil {
return err
}
immetrics.KafkaMessagePersistTotal.WithLabelValues(kind, "single_ok").Inc()
immetrics.MessagePersistDuration.WithLabelValues(kind, "single_ok").Observe(time.Since(started).Seconds())
return nil
}
markFailed := func(err error) (*types.KafkaConsumeRecord, error) {
immetrics.KafkaMessagePersistTotal.WithLabelValues(kind, "retry").Inc()
return store.HTTPInfrastructure.MarkKafkaConsumeFailed(item.tenantID, item.envelope.EventID, kafkaBus.config.ConsumerGroup,
kafkaMessagePersistFailureReason(err), err.Error(), time.Now().UTC().Round(time.Millisecond))
}
markDead := func(failed *types.KafkaConsumeRecord, err error) error {
return markKafkaDeadLetter(ctx, item.tenantID, item.envelope, failed, kafkaMessagePersistFailureReason(err), err.Error())
immetrics.KafkaMessagePersistTotal.WithLabelValues(kind, "dead").Inc()
immetrics.MessagePersistDuration.WithLabelValues(kind, "dead").Observe(time.Since(started).Seconds())
reason := kafkaMessagePersistFailureReason(err)
immetrics.KafkaDeadLettersTotal.WithLabelValues(kind, reason).Inc()
return markKafkaDeadLetter(ctx, item.tenantID, item.envelope, failed, reason, err.Error())
}
return runKafkaMessageSingleRetries(ctx, kafkaBus.config.MessageRetries, operation, markFailed, markDead, waitKafkaRetry)
}