pro/im_server
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(server): Kafka消息持久化重试耗尽时死信处理避免无限重试

Browse Source 
在Kafka消息持久化重试耗尽时，新增将消息死信的逻辑，而非保留未提交的偏移量，避免无法解决的故障导致分区消息无限重试阻塞后续消息。同时补充死信处理的日志，确保问题可追踪。
This commit is contained in:
Austin
2026-10-13 15:40:19 +08:00
parent f3b1a06da5
commit b126b13b77
2 changed files with 31 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/kafka_message_persist.go
+15 -5
View File
@@ -159,6 +159,7 @@ type kafkaMessagePersistItem struct {
envelope kafkaEnvelope
tenantID types.TenantID
payload kafkaMessagePersistPayload
consume *types.KafkaConsumeRecord
}
type kafkaMessagePersistPreparer func(context.Context, kafka.Message, string) (*kafkaMessagePersistItem, bool, error)
@@ -259,7 +260,7 @@ func prepareKafkaMessagePersistItem(ctx context.Context, message kafka.Message,
}
return nil, true, nil
}
item := &kafkaMessagePersistItem{envelope: envelope, tenantID: tenantID}
item := &kafkaMessagePersistItem{envelope: envelope, tenantID: tenantID, consume: consume}
if err = json.Unmarshal(envelope.Payload, &item.payload); err != nil {
return nil, true, markKafkaDeadLetter(ctx, tenantID, envelope, consume, "MALFORMED_PAYLOAD", err.Error())
}
@@ -308,14 +309,23 @@ func persistKafkaMessageIndividually(ctx context.Context, item *kafkaMessagePers
}
err := runKafkaMessageSingleRetries(ctx, kafkaBus.config.MessageRetries, operation, markFailed, waitKafkaRetry)
if err != nil {
// Message persistence failures are recoverable infrastructure failures. Returning
// the error keeps the Kafka offset uncommitted so the outer consumer retries the
// message until it is durable. Only malformed payloads are dead-lettered while
// preparing the item.
immetrics.KafkaMessagePersistTotal.WithLabelValues(kind, "retry_exhausted").Inc()
immetrics.MessagePersistDuration.WithLabelValues(kind, "retry_exhausted").Observe(time.Since(started).Seconds())
logs.Err.Printf("kafka message persist retry exhausted: tenant=%d event=%s topic=%s seq=%d kind=%s err=%v",
item.tenantID, item.envelope.EventID, item.payload.Message.Topic, item.payload.Message.SeqId, kind, err)
// Dead-letter the message instead of leaving the offset uncommitted: an
// unresolvable failure (e.g. an attachment file already reclaimed by GC)
// would otherwise be retried forever and stall every message behind it
// on the partition. The dead letter keeps the envelope for replay once
// the underlying issue is fixed.
if deadErr := markKafkaDeadLetter(ctx, item.tenantID, item.envelope, item.consume, "PERSIST_RETRY_EXHAUSTED", err.Error()); deadErr != nil {
logs.Err.Printf("kafka message persist dead letter failed, keeping offset uncommitted: tenant=%d event=%s topic=%s seq=%d err=%v",
item.tenantID, item.envelope.EventID, item.payload.Message.Topic, item.payload.Message.SeqId, deadErr)
return err
}
logs.Warn.Printf("kafka message persist dead-lettered after retries: tenant=%d event=%s topic=%s seq=%d kind=%s err=%v",
item.tenantID, item.envelope.EventID, item.payload.Message.Topic, item.payload.Message.SeqId, kind, err)
return nil
}
return err
}