feat: batch kafka message persistence

server/kafka_config_events.go

@@ -3,6 +3,7 @@ package main
 import (
 	"context"
 	"encoding/json"
+	"errors"
 	"strings"
 	"time"
 
@@ -12,6 +13,8 @@ import (
 	"github.com/tinode/chat/server/store/types"
 )
 
+var errKafkaConsumeInProgress = errors.New("kafka event is already being processed")
+
 const (
 	configEventTenantUpdated         = "tenant.updated"
 	configEventTenantStatusChanged   = "tenant.status.changed"
@@ -66,7 +69,6 @@ func handleKafkaConfigEvent(ctx context.Context, envelope kafkaEnvelope, _ kafka
 		if failed.RetryCount >= kafkaBus.config.MaxRetryCount {
 			return markKafkaDeadLetter(ctx, tenantID, envelope, failed, reason, err.Error())
 		}
-		sleepKafkaRetryBackoff(failed.RetryCount)
 		return err
 	}
 	return store.HTTPInfrastructure.MarkKafkaConsumeSucceeded(tenantID, envelope.EventID, consumerGroup, time.Now().UTC().Round(time.Millisecond))
@@ -155,7 +157,7 @@ func beginKafkaConsumeForTopicWithGroup(tenantID types.TenantID, envelope kafkaE
 		consumerGroup = kafkaBus.config.ConsumerGroup
 	}
 	deadline := time.Now().UTC().Add(5 * time.Minute).Round(time.Millisecond)
-	return store.HTTPInfrastructure.BeginKafkaConsume(tenantID, &types.KafkaConsumeRecord{
+	record, claimed, err := store.HTTPInfrastructure.BeginKafkaConsume(tenantID, &types.KafkaConsumeRecord{
 		EventID:       envelope.EventID,
 		ConsumerGroup: consumerGroup,
 		SourceTopic:   kafkaTopic(topicName),
@@ -163,4 +165,11 @@ func beginKafkaConsumeForTopicWithGroup(tenantID types.TenantID, envelope kafkaE
 		BizType:       envelope.BizType,
 		BizID:         envelope.BizID,
 	}, deadline)
+	if err != nil || claimed || record == nil {
+		return record, claimed, err
+	}
+	if record.Status != types.KafkaConsumeSucceeded && record.Status != types.KafkaConsumeDead {
+		return record, false, errKafkaConsumeInProgress
+	}
+	return record, false, nil
 }