feat: batch kafka message persistence

This commit is contained in:
richjohnrambo
2026-09-24 22:45:07 +08:00
parent 7f2df1dd4f
commit f898a7171c
18 changed files with 682 additions and 98 deletions
+7 -18
View File
@@ -141,7 +141,6 @@ func handleKafkaAdminCommand(ctx context.Context, envelope kafkaEnvelope, _ kafk
return nil
}
if kafkaCommandReasonRetryable(reason) {
sleepKafkaRetryBackoff(failedRecord.RetryCount)
return execErr
}
if err := publishKafkaAdminCommandResult(ctx, envelope, payload.CommandID, "failed", reason, execErr.Error(), result); err != nil {
@@ -608,7 +607,7 @@ func markKafkaDeadLetter(ctx context.Context, tenantID types.TenantID, envelope
if err != nil {
return err
}
return publishKafkaDeadLetter(ctx, "dead_letter", envelope.EventID, kafkaEnvelope{
if err = publishKafkaDeadLetter(ctx, "dead_letter", envelope.EventID, kafkaEnvelope{
EventID: deadID,
TraceID: envelope.TraceID,
TenantCode: envelope.TenantCode,
@@ -621,7 +620,12 @@ func markKafkaDeadLetter(ctx context.Context, tenantID types.TenantID, envelope
OccurredAt: now,
ParentEventID: envelope.EventID,
Payload: deadPayload,
})
}); err != nil {
return err
}
// Persist the terminal state before the source consumer commits its offset.
// The dead-letter consumer repeats this idempotent upsert for recovery and auditing.
return store.HTTPInfrastructure.MarkKafkaConsumeDead(tenantID, dead, now)
}
func shouldDeadLetterKafkaCommand(reason string, retryCount int) bool {
@@ -643,21 +647,6 @@ func kafkaCommandReasonRetryable(reason string) bool {
}
}
func sleepKafkaRetryBackoff(retryCount int) {
if kafkaBus == nil || retryCount <= 0 {
return
}
backoffs := kafkaBus.config.RetryBackoffMS
index := retryCount - 1
if index >= len(backoffs) {
index = len(backoffs) - 1
}
delay := time.Duration(backoffs[index]) * time.Millisecond
if delay > 0 {
time.Sleep(delay)
}
}
func parseKafkaCommandUser(raw string) types.Uid {
raw = strings.TrimSpace(raw)
if raw == "" {