pro/im_server
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(kafka): retain failed message persistence

Browse Source
This commit is contained in:
richjohnrambo
2026-10-03 22:32:08 +08:00
parent db92c41241
commit 746f75ab0f
9 changed files with 85 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/kafka_message_persist.go
+12 -12
View File
@@ -281,27 +281,27 @@ func persistKafkaMessageIndividually(ctx context.Context, item *kafkaMessagePers
return store.HTTPInfrastructure.MarkKafkaConsumeFailed(item.tenantID, item.envelope.EventID, kafkaBus.config.ConsumerGroup,
kafkaMessagePersistFailureReason(err), err.Error(), time.Now().UTC().Round(time.Millisecond))
}
markDead := func(failed *types.KafkaConsumeRecord, err error) error {
immetrics.KafkaMessagePersistTotal.WithLabelValues(kind, "dead").Inc()
immetrics.MessagePersistDuration.WithLabelValues(kind, "dead").Observe(time.Since(started).Seconds())
reason := kafkaMessagePersistFailureReason(err)
immetrics.KafkaDeadLettersTotal.WithLabelValues(kind, reason).Inc()
return markKafkaDeadLetter(ctx, item.tenantID, item.envelope, failed, reason, err.Error())
err := runKafkaMessageSingleRetries(ctx, kafkaBus.config.MessageRetries, operation, markFailed, waitKafkaRetry)
if err != nil {
// Message persistence failures are recoverable infrastructure failures. Returning
// the error keeps the Kafka offset uncommitted so the outer consumer retries the
// message until it is durable. Only malformed payloads are dead-lettered while
// preparing the item.
immetrics.KafkaMessagePersistTotal.WithLabelValues(kind, "retry_exhausted").Inc()
immetrics.MessagePersistDuration.WithLabelValues(kind, "retry_exhausted").Observe(time.Since(started).Seconds())
}
return runKafkaMessageSingleRetries(ctx, kafkaBus.config.MessageRetries, operation, markFailed, markDead, waitKafkaRetry)
return err
}
func runKafkaMessageSingleRetries(ctx context.Context, retryCount int, operation func() error,
markFailed func(error) (*types.KafkaConsumeRecord, error), markDead func(*types.KafkaConsumeRecord, error) error,
wait kafkaRetryWaiter) error {
markFailed func(error) (*types.KafkaConsumeRecord, error), wait kafkaRetryWaiter) error {
var lastErr error
var failed *types.KafkaConsumeRecord
for attempt := 1; attempt <= retryCount+1; attempt++ {
if lastErr = operation(); lastErr == nil {
return nil
}
var err error
if failed, err = markFailed(lastErr); err != nil {
if _, err = markFailed(lastErr); err != nil {
return err
}
if attempt <= retryCount {
@@ -310,7 +310,7 @@ func runKafkaMessageSingleRetries(ctx context.Context, retryCount int, operation
}
}
}
return markDead(failed, lastErr)
return fmt.Errorf("message persistence retries exhausted: %w", lastErr)
}
func finishKafkaMessagePersist(ctx context.Context, item *kafkaMessagePersistItem) error {