feat: batch kafka message persistence

server/db/adapter.go

@@ -260,6 +260,11 @@ type TenantRoutedAdapter interface {
 	TenantAdapterFor(t.TenantID) (TenantBusinessAdapter, TenantAuthAdapter, error)
 }
 
+type TenantMessageBatchAdapter interface {
+	TenantMessageSaveBatch(t.TenantID, []*t.Message) error
+	TenantGroupMessageSaveBatch(t.TenantID, []*t.Message) error
+}
+
 // TenantHTTPInfrastructureAdapter is the optional durable infrastructure used
 // by Kafka dead-letter handling.
 type TenantHTTPInfrastructureAdapter interface {

server/db/mysql/tenant_adapter.go

@@ -2249,6 +2249,41 @@ func (a *adapter) TenantMessageSave(id t.TenantID, msg *t.Message) error {
 	return err
 }
 
+func (a *adapter) TenantMessageSaveBatch(id t.TenantID, messages []*t.Message) error {
+	tenantAdp, err := a.routedTenantAdapter(id)
+	if err != nil {
+		return err
+	}
+	if len(messages) == 0 {
+		return t.ErrMalformed
+	}
+	if tenantAdp != a {
+		return tenantAdp.TenantMessageSaveBatch(id, messages)
+	}
+	ctx, cancel := a.getContext()
+	if cancel != nil {
+		defer cancel()
+	}
+	values := make([]string, 0, len(messages))
+	args := make([]any, 0, len(messages)*13)
+	for _, msg := range messages {
+		if msg == nil || msg.TenantID != id || msg.Topic == "" || msg.SeqId <= 0 || msg.Uid().IsZero() {
+			return t.ErrMalformed
+		}
+		values = append(values, "(?,?,?,?,?,?,?,?,?,?,?,?,?)")
+		args = append(args, id, store.DecodeUid(msg.Uid()), msg.CreatedAt, msg.UpdatedAt, msg.SeqId, msg.Topic,
+			decodeUID(t.ParseUid(msg.From)), sql.NullString{String: msg.ClientMsgID, Valid: msg.ClientMsgID != ""},
+			messageVersion(msg.Version), msg.Head, common.ToJSON(msg.Content), msg.ContentNonce, msg.KeyVersion)
+	}
+	query := fmt.Sprintf("INSERT INTO %s(tenant_id,id,createdat,updatedat,seqid,topic,`from`,client_msg_id,version,head,content,content_nonce,key_version) VALUES %s",
+		tenantMessagesTable(id), strings.Join(values, ","))
+	_, err = a.db.ExecContext(ctx, query, args...)
+	if isDupe(err) {
+		return t.ErrDuplicate
+	}
+	return err
+}
+
 func (a *adapter) TenantGroupMessageSave(id t.TenantID, msg *t.Message) error {
 	tenantAdp, err := a.routedTenantAdapter(id)
 	if err != nil || msg == nil || msg.TenantID != id || msg.Topic == "" || msg.SeqId <= 0 {
@@ -2315,6 +2350,69 @@ func (a *adapter) TenantGroupMessageSave(id t.TenantID, msg *t.Message) error {
 	return tx.Commit()
 }
 
+func (a *adapter) TenantGroupMessageSaveBatch(id t.TenantID, messages []*t.Message) error {
+	tenantAdp, err := a.routedTenantAdapter(id)
+	if err != nil {
+		return err
+	}
+	if len(messages) == 0 {
+		return t.ErrMalformed
+	}
+	if tenantAdp != a {
+		return tenantAdp.TenantGroupMessageSaveBatch(id, messages)
+	}
+	if err = a.ensureCurrentGroupMessageShardWindow(); err != nil {
+		return err
+	}
+	ctx, cancel := a.getContext()
+	if cancel != nil {
+		defer cancel()
+	}
+	month := messages[0].ShardMonth
+	table, ok := groupMessageShardTable(id, month)
+	if !ok {
+		return t.ErrMalformed
+	}
+	groupIDs := make(map[string]int64)
+	for _, msg := range messages {
+		if msg == nil || msg.TenantID != id || msg.Topic == "" || msg.SeqId <= 0 || msg.Uid().IsZero() || msg.ShardMonth != month {
+			return t.ErrMalformed
+		}
+		if _, found := groupIDs[msg.Topic]; !found {
+			groupIDs[msg.Topic], err = a.groupIDByTopic(ctx, id, msg.Topic)
+			if err != nil {
+				return err
+			}
+		}
+	}
+	var shardStatus string
+	if err = a.db.QueryRowContext(ctx, "SELECT status FROM im_group_message_shards WHERE tenant_id=? AND shard_month=? AND table_name=?", id, month, table).Scan(&shardStatus); err != nil {
+		return err
+	}
+	if shardStatus != "writable" && shardStatus != "sealing" {
+		return fmt.Errorf("mysql: group message shard %s is %s", month, shardStatus)
+	}
+	values := make([]string, 0, len(messages))
+	args := make([]any, 0, len(messages)*20)
+	for _, msg := range messages {
+		contentJSON := common.ToJSON(msg.Content)
+		values = append(values, "(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)")
+		args = append(args, id, store.DecodeUid(msg.Uid()), msg.Topic, groupIDs[msg.Topic], msg.SeqId,
+			decodeUID(t.ParseUid(msg.From)), sql.NullString{String: msg.ClientMsgID, Valid: msg.ClientMsgID != ""},
+			messageHeadType(msg.Head), "normal", contentJSON, msg.Head, messageHeadJSON(msg.Head, "mentioned_user_ids"),
+			messageHeadBool(msg.Head, "mention_all"), 0, len(contentJSON), messageVersion(msg.Version), msg.CreatedAt,
+			msg.UpdatedAt, msg.ContentNonce, msg.KeyVersion)
+	}
+	query := fmt.Sprintf(`INSERT INTO %s
+		(tenant_id,message_id,topic,group_id,seqid,from_user,client_msg_id,msg_type,status,content,head,mentioned_user_ids,mention_all,
+		 file_count,content_size,version,created_at,updated_at,content_nonce,key_version) VALUES %s`, table, strings.Join(values, ","))
+	_, err = a.db.ExecContext(ctx, query, args...)
+	if isDupe(err) {
+		return t.ErrDuplicate
+	}
+	return err
+}
+
 func (a *adapter) TenantMessageGetByClientID(id t.TenantID, from t.Uid, clientMsgID string) (*t.Message, error) {
 	if !a.tenantScopedDB {
 		tenantAdp, err := a.routedTenantAdapter(id)