fix(kafka): preserve partially fetched message batches

server/kafka_message_persist.go

@@ -161,11 +161,26 @@ type kafkaMessagePersistItem struct {
 	payload  kafkaMessagePersistPayload
 }
 
+type kafkaMessagePersistPreparer func(context.Context, kafka.Message, string) (*kafkaMessagePersistItem, bool, error)
+type kafkaMessagePersistGroupsWriter func(context.Context, map[string][]*kafkaMessagePersistItem) error
+
 func handleKafkaMessagePersistBatch(ctx context.Context, messages []kafka.Message, sourceTopicName string) error {
+	return processKafkaMessagePersistBatch(ctx, messages, sourceTopicName,
+		prepareKafkaMessagePersistItem, persistKafkaMessagePersistGroups)
+}
+
+func processKafkaMessagePersistBatch(ctx context.Context, messages []kafka.Message, sourceTopicName string,
+	prepare kafkaMessagePersistPreparer, persist kafkaMessagePersistGroupsWriter) error {
 	groups := make(map[string][]*kafkaMessagePersistItem)
 	for _, message := range messages {
-		item, terminal, err := prepareKafkaMessagePersistItem(ctx, message, sourceTopicName)
+		item, terminal, err := prepare(ctx, message, sourceTopicName)
 		if err != nil {
+			// Preparing an item acquires its idempotency claim. If a later item
+			// fails, finish all earlier claimed items before retrying the batch;
+			// otherwise retries can keep colliding with those processing claims.
+			if flushErr := persist(ctx, groups); flushErr != nil {
+				return fmt.Errorf("persist claimed Kafka messages after prepare failure: %w (prepare error: %v)", flushErr, err)
+			}
 			return err
 		}
 		if terminal {
@@ -177,7 +192,14 @@ func handleKafkaMessagePersistBatch(ctx context.Context, messages []kafka.Messag
 		}
 		groups[key] = append(groups[key], item)
 	}
+	return persist(ctx, groups)
+}
+
+func persistKafkaMessagePersistGroups(ctx context.Context, groups map[string][]*kafkaMessagePersistItem) error {
 	for _, items := range groups {
+		if len(items) == 0 {
+			continue
+		}
 		kind := items[0].payload.MessageKind
 		if kind == "" {
 			kind = "p2p"