feat: add kafka admin integration

2026-08-28 00:05:54 +08:00
parent b2e5097708
commit 5cc6eebc1a
48 changed files with 8065 additions and 55 deletions
@@ -4,8 +4,11 @@
 package mysql

 import (
+	"context"
 	"database/sql"
 	"encoding/json"
+	"strconv"
+	"strings"
 	"time"

 	"github.com/jmoiron/sqlx"
@@ -225,6 +228,512 @@ func (a *adapter) TenantBusinessEventMarkDelivered(id t.TenantID, eventID string
 	return nil
 }

+func (a *adapter) TenantKafkaConsumeBegin(id t.TenantID, record *t.KafkaConsumeRecord, deadline time.Time) (*t.KafkaConsumeRecord, bool, error) {
+	tenantAdp, err := a.infrastructureAdapter(id)
+	if err != nil {
+		return nil, false, err
+	}
+	if record == nil || record.EventID == "" || record.ConsumerGroup == "" || record.SourceTopic == "" {
+		return nil, false, t.ErrMalformed
+	}
+	if tenantAdp != a {
+		return tenantAdp.TenantKafkaConsumeBegin(id, record, deadline)
+	}
+	now := t.TimeNow()
+	_, err = a.db.Exec(`INSERT INTO im_kafka_consume_log
+		(event_id,consumer_group,source_topic,event_type,biz_type,biz_id,status,processing_deadline,created_at,updated_at)
+		VALUES(?,?,?,?,?,?,?,?,?,?)`, record.EventID, record.ConsumerGroup, record.SourceTopic, record.EventType,
+		record.BizType, record.BizID, t.KafkaConsumeProcessing, deadline, now, now)
+	if err == nil {
+		record.Status = t.KafkaConsumeProcessing
+		record.ProcessingDeadline = &deadline
+		record.CreatedAt = now
+		record.UpdatedAt = now
+		return record, true, nil
+	}
+	if !isDupe(err) {
+		return nil, false, err
+	}
+	existing, err := a.kafkaConsumeGet(record.EventID, record.ConsumerGroup)
+	if err != nil {
+		return nil, false, err
+	}
+	if existing.Status == t.KafkaConsumeSucceeded || existing.Status == t.KafkaConsumeDead {
+		return existing, false, nil
+	}
+	if existing.Status == t.KafkaConsumeProcessing && existing.ProcessingDeadline != nil && existing.ProcessingDeadline.After(now) {
+		return existing, false, nil
+	}
+	res, err := a.db.Exec(`UPDATE im_kafka_consume_log SET status=?,source_topic=?,event_type=?,biz_type=?,biz_id=?,
+		processing_deadline=?,updated_at=? WHERE event_id=? AND consumer_group=? AND status<>?`,
+		t.KafkaConsumeProcessing, record.SourceTopic, record.EventType, record.BizType, record.BizID,
+		deadline, now, record.EventID, record.ConsumerGroup, t.KafkaConsumeSucceeded)
+	if err != nil {
+		return nil, false, err
+	}
+	if affected, _ := res.RowsAffected(); affected == 0 {
+		current, getErr := a.kafkaConsumeGet(record.EventID, record.ConsumerGroup)
+		return current, false, getErr
+	}
+	current, err := a.kafkaConsumeGet(record.EventID, record.ConsumerGroup)
+	return current, err == nil, err
+}
+
+func (a *adapter) TenantKafkaConsumeSucceed(id t.TenantID, eventID, consumerGroup string, now time.Time) error {
+	tenantAdp, err := a.infrastructureAdapter(id)
+	if err != nil {
+		return err
+	}
+	if eventID == "" || consumerGroup == "" {
+		return t.ErrMalformed
+	}
+	if tenantAdp != a {
+		return tenantAdp.TenantKafkaConsumeSucceed(id, eventID, consumerGroup, now)
+	}
+	res, err := a.db.Exec(`UPDATE im_kafka_consume_log SET status=?,succeeded_at=?,processing_deadline=NULL,updated_at=?
+		WHERE event_id=? AND consumer_group=?`, t.KafkaConsumeSucceeded, now, now, eventID, consumerGroup)
+	if err != nil {
+		return err
+	}
+	if affected, _ := res.RowsAffected(); affected == 0 {
+		return t.ErrNotFound
+	}
+	return nil
+}
+
+func (a *adapter) TenantKafkaConsumeFail(id t.TenantID, eventID, consumerGroup, reason, message string, now time.Time) (*t.KafkaConsumeRecord, error) {
+	tenantAdp, err := a.infrastructureAdapter(id)
+	if err != nil {
+		return nil, err
+	}
+	if eventID == "" || consumerGroup == "" {
+		return nil, t.ErrMalformed
+	}
+	if tenantAdp != a {
+		return tenantAdp.TenantKafkaConsumeFail(id, eventID, consumerGroup, reason, message, now)
+	}
+	_, err = a.db.Exec(`UPDATE im_kafka_consume_log SET status=?,retry_count=retry_count+1,last_reason=?,
+		last_error=?,processing_deadline=NULL,updated_at=? WHERE event_id=? AND consumer_group=?`,
+		t.KafkaConsumeFailed, truncateKafkaString(reason, 64), truncateKafkaString(message, 1000), now, eventID, consumerGroup)
+	if err != nil {
+		return nil, err
+	}
+	return a.kafkaConsumeGet(eventID, consumerGroup)
+}
+
+func (a *adapter) TenantKafkaConsumeDead(id t.TenantID, dead *t.KafkaDeadLetter, now time.Time) error {
+	tenantAdp, err := a.infrastructureAdapter(id)
+	if err != nil {
+		return err
+	}
+	if dead == nil || dead.DeadID == "" || dead.EventID == "" || dead.ConsumerGroup == "" || dead.SourceTopic == "" || len(dead.Envelope) == 0 {
+		return t.ErrMalformed
+	}
+	if tenantAdp != a {
+		return tenantAdp.TenantKafkaConsumeDead(id, dead, now)
+	}
+	tx, err := a.db.BeginTxx(context.Background(), nil)
+	if err != nil {
+		return err
+	}
+	defer tx.Rollback()
+	if _, err = tx.Exec(`INSERT INTO im_kafka_dead_letters
+		(dead_id,event_id,parent_event_id,source_topic,consumer_group,event_type,biz_type,biz_id,envelope,
+		 retry_count,last_reason,last_error,status,replay_count,created_at,updated_at)
+		VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)
+		ON DUPLICATE KEY UPDATE last_reason=VALUES(last_reason),last_error=VALUES(last_error),updated_at=VALUES(updated_at)`,
+		dead.DeadID, dead.EventID, dead.ParentEventID, dead.SourceTopic, dead.ConsumerGroup, dead.EventType,
+		dead.BizType, dead.BizID, dead.Envelope, dead.RetryCount, truncateKafkaString(dead.LastReason, 64),
+		truncateKafkaString(dead.LastError, 1000), t.KafkaDeadStatusDead, dead.ReplayCount, now, now); err != nil {
+		return err
+	}
+	if _, err = tx.Exec(`UPDATE im_kafka_consume_log SET status=?,dead_at=?,processing_deadline=NULL,last_reason=?,
+		last_error=?,updated_at=? WHERE event_id=? AND consumer_group=?`,
+		t.KafkaConsumeDead, now, truncateKafkaString(dead.LastReason, 64), truncateKafkaString(dead.LastError, 1000),
+		now, dead.EventID, dead.ConsumerGroup); err != nil {
+		return err
+	}
+	return tx.Commit()
+}
+
+func (a *adapter) TenantKafkaDeadLetterList(id t.TenantID, opts t.KafkaDeadLetterListOptions) (*t.KafkaDeadLetterPage, error) {
+	tenantAdp, err := a.infrastructureAdapter(id)
+	if err != nil {
+		return nil, err
+	}
+	if tenantAdp != a {
+		return tenantAdp.TenantKafkaDeadLetterList(id, opts)
+	}
+	limit := opts.Limit
+	if limit <= 0 || limit > 200 {
+		limit = 50
+	}
+	where := []string{"1=1"}
+	args := []any{}
+	if opts.Status != "" {
+		where = append(where, "status=?")
+		args = append(args, opts.Status)
+	}
+	if opts.BizType != "" {
+		where = append(where, "biz_type=?")
+		args = append(args, opts.BizType)
+	}
+	if opts.BizID != "" {
+		where = append(where, "biz_id=?")
+		args = append(args, opts.BizID)
+	}
+	if opts.EventType != "" {
+		where = append(where, "event_type=?")
+		args = append(args, opts.EventType)
+	}
+	if opts.Cursor != "" {
+		where = append(where, "id<?")
+		args = append(args, opts.Cursor)
+	}
+	args = append(args, limit+1)
+	rows, err := a.db.Queryx(`SELECT id,dead_id,event_id,parent_event_id,source_topic,consumer_group,event_type,biz_type,biz_id,
+		envelope,retry_count,last_reason,last_error,status,replay_count,created_at,updated_at
+		FROM im_kafka_dead_letters WHERE `+strings.Join(where, " AND ")+` ORDER BY id DESC LIMIT ?`, args...)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	page := &t.KafkaDeadLetterPage{Items: make([]t.KafkaDeadLetter, 0, limit)}
+	for rows.Next() {
+		dead, err := scanKafkaDeadLetter(rows)
+		if err != nil {
+			return nil, err
+		}
+		if len(page.Items) == limit {
+			page.NextCursor = strconv.FormatInt(dead.ID, 10)
+			continue
+		}
+		page.Items = append(page.Items, *dead)
+	}
+	return page, rows.Err()
+}
+
+func (a *adapter) TenantKafkaDeadLetterGet(id t.TenantID, deadID string) (*t.KafkaDeadLetter, error) {
+	tenantAdp, err := a.infrastructureAdapter(id)
+	if err != nil {
+		return nil, err
+	}
+	if strings.TrimSpace(deadID) == "" {
+		return nil, t.ErrMalformed
+	}
+	if tenantAdp != a {
+		return tenantAdp.TenantKafkaDeadLetterGet(id, deadID)
+	}
+	rows, err := a.db.Queryx(`SELECT id,dead_id,event_id,parent_event_id,source_topic,consumer_group,event_type,biz_type,biz_id,
+		envelope,retry_count,last_reason,last_error,status,replay_count,created_at,updated_at
+		FROM im_kafka_dead_letters WHERE dead_id=?`, strings.TrimSpace(deadID))
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	if rows.Next() {
+		return scanKafkaDeadLetter(rows)
+	}
+	if err = rows.Err(); err != nil {
+		return nil, err
+	}
+	return nil, t.ErrNotFound
+}
+
+func (a *adapter) TenantKafkaDeadLetterMark(id t.TenantID, deadID, status, resolvedBy, note string, now time.Time) error {
+	tenantAdp, err := a.infrastructureAdapter(id)
+	if err != nil {
+		return err
+	}
+	deadID = strings.TrimSpace(deadID)
+	status = strings.TrimSpace(status)
+	if deadID == "" || (status != t.KafkaDeadStatusIgnored && status != t.KafkaDeadStatusResolved && status != t.KafkaDeadStatusReplayed) {
+		return t.ErrMalformed
+	}
+	if tenantAdp != a {
+		return tenantAdp.TenantKafkaDeadLetterMark(id, deadID, status, resolvedBy, note, now)
+	}
+	res, err := a.db.Exec(`UPDATE im_kafka_dead_letters SET status=?,resolved_by=?,resolved_at=?,resolve_note=?,updated_at=?
+		WHERE dead_id=? AND status IN (?,?,?)`, status, truncateKafkaString(resolvedBy, 64), now,
+		truncateKafkaString(note, 1000), now, deadID, t.KafkaDeadStatusDead, t.KafkaDeadStatusPendingReplay, t.KafkaDeadStatusReplayed)
+	if err != nil {
+		return err
+	}
+	if affected, _ := res.RowsAffected(); affected == 0 {
+		return t.ErrNotFound
+	}
+	return nil
+}
+
+func (a *adapter) TenantKafkaDeadLetterMarkReplay(id t.TenantID, deadID, resolvedBy, note string, now time.Time) (*t.KafkaDeadLetter, error) {
+	tenantAdp, err := a.infrastructureAdapter(id)
+	if err != nil {
+		return nil, err
+	}
+	deadID = strings.TrimSpace(deadID)
+	if deadID == "" {
+		return nil, t.ErrMalformed
+	}
+	if tenantAdp != a {
+		return tenantAdp.TenantKafkaDeadLetterMarkReplay(id, deadID, resolvedBy, note, now)
+	}
+	tx, err := a.db.BeginTxx(context.Background(), nil)
+	if err != nil {
+		return nil, err
+	}
+	defer tx.Rollback()
+	var row struct {
+		EventID       string `db:"event_id"`
+		ConsumerGroup string `db:"consumer_group"`
+	}
+	if err = tx.Get(&row, `SELECT event_id,consumer_group FROM im_kafka_dead_letters WHERE dead_id=? FOR UPDATE`, deadID); err != nil {
+		if err == sql.ErrNoRows {
+			return nil, t.ErrNotFound
+		}
+		return nil, err
+	}
+	res, err := tx.Exec(`UPDATE im_kafka_dead_letters SET status=?,replay_count=replay_count+1,resolved_by=?,
+		resolved_at=?,resolve_note=?,updated_at=? WHERE dead_id=? AND status IN (?,?)`,
+		t.KafkaDeadStatusPendingReplay, truncateKafkaString(resolvedBy, 64), now, truncateKafkaString(note, 1000),
+		now, deadID, t.KafkaDeadStatusDead, t.KafkaDeadStatusPendingReplay)
+	if err != nil {
+		return nil, err
+	}
+	if affected, _ := res.RowsAffected(); affected == 0 {
+		return nil, t.ErrNotFound
+	}
+	if _, err = tx.Exec(`DELETE FROM im_kafka_consume_log WHERE event_id=? AND consumer_group=?`,
+		row.EventID, row.ConsumerGroup); err != nil {
+		return nil, err
+	}
+	if err = tx.Commit(); err != nil {
+		return nil, err
+	}
+	return a.TenantKafkaDeadLetterGet(id, deadID)
+}
+
+func (a *adapter) TenantKafkaOutboxCreate(id t.TenantID, record *t.KafkaOutboxRecord) error {
+	tenantAdp, err := a.infrastructureAdapter(id)
+	if err != nil {
+		return err
+	}
+	if record == nil || record.EventID == "" || record.Topic == "" || record.EventType == "" || len(record.Envelope) == 0 {
+		return t.ErrMalformed
+	}
+	if tenantAdp != a {
+		return tenantAdp.TenantKafkaOutboxCreate(id, record)
+	}
+	now := t.TimeNow()
+	_, err = a.db.Exec(`INSERT INTO im_kafka_outbox
+		(event_id,topic,event_type,biz_type,biz_id,key_value,envelope,status,next_retry_at,created_at,updated_at)
+		VALUES(?,?,?,?,?,?,?,?,?,?,?)
+		ON DUPLICATE KEY UPDATE updated_at=updated_at`,
+		record.EventID, record.Topic, record.EventType, record.BizType, record.BizID, record.Key,
+		record.Envelope, t.KafkaOutboxPending, record.NextRetryAt, now, now)
+	return err
+}
+
+func (a *adapter) TenantKafkaOutboxListPending(id t.TenantID, limit int, now time.Time) ([]t.KafkaOutboxRecord, error) {
+	tenantAdp, err := a.infrastructureAdapter(id)
+	if err != nil {
+		return nil, err
+	}
+	if limit <= 0 {
+		limit = 100
+	}
+	if tenantAdp != a {
+		return tenantAdp.TenantKafkaOutboxListPending(id, limit, now)
+	}
+	rows := []struct {
+		ID          int64        `db:"id"`
+		EventID     string       `db:"event_id"`
+		Topic       string       `db:"topic"`
+		EventType   string       `db:"event_type"`
+		BizType     string       `db:"biz_type"`
+		BizID       string       `db:"biz_id"`
+		Key         string       `db:"key_value"`
+		Envelope    []byte       `db:"envelope"`
+		Status      string       `db:"status"`
+		Attempts    int          `db:"attempts"`
+		NextRetryAt sql.NullTime `db:"next_retry_at"`
+		SentAt      sql.NullTime `db:"sent_at"`
+		LastReason  string       `db:"last_reason"`
+		LastError   string       `db:"last_error"`
+		CreatedAt   time.Time    `db:"created_at"`
+		UpdatedAt   time.Time    `db:"updated_at"`
+	}{}
+	err = a.db.Select(&rows, `SELECT id,event_id,topic,event_type,biz_type,biz_id,key_value,envelope,status,attempts,
+		next_retry_at,sent_at,last_reason,last_error,created_at,updated_at
+		FROM im_kafka_outbox
+		WHERE status IN (?,?) AND (next_retry_at IS NULL OR next_retry_at<=?)
+		ORDER BY id ASC LIMIT ?`, t.KafkaOutboxPending, t.KafkaOutboxFailed, now, limit)
+	if err != nil {
+		return nil, err
+	}
+	out := make([]t.KafkaOutboxRecord, 0, len(rows))
+	for _, row := range rows {
+		rec := t.KafkaOutboxRecord{
+			ID:         row.ID,
+			EventID:    row.EventID,
+			Topic:      row.Topic,
+			EventType:  row.EventType,
+			BizType:    row.BizType,
+			BizID:      row.BizID,
+			Key:        row.Key,
+			Envelope:   row.Envelope,
+			Status:     row.Status,
+			Attempts:   row.Attempts,
+			LastReason: row.LastReason,
+			LastError:  row.LastError,
+			CreatedAt:  row.CreatedAt,
+			UpdatedAt:  row.UpdatedAt,
+		}
+		if row.NextRetryAt.Valid {
+			rec.NextRetryAt = &row.NextRetryAt.Time
+		}
+		if row.SentAt.Valid {
+			rec.SentAt = &row.SentAt.Time
+		}
+		out = append(out, rec)
+	}
+	return out, nil
+}
+
+func (a *adapter) TenantKafkaOutboxMarkSent(id t.TenantID, eventID string, now time.Time) error {
+	tenantAdp, err := a.infrastructureAdapter(id)
+	if err != nil {
+		return err
+	}
+	if eventID == "" {
+		return t.ErrMalformed
+	}
+	if tenantAdp != a {
+		return tenantAdp.TenantKafkaOutboxMarkSent(id, eventID, now)
+	}
+	res, err := a.db.Exec(`UPDATE im_kafka_outbox SET status=?,sent_at=?,attempts=attempts+1,last_reason='',
+		last_error='',updated_at=? WHERE event_id=? AND status<>?`,
+		t.KafkaOutboxSent, now, now, eventID, t.KafkaOutboxSent)
+	if err != nil {
+		return err
+	}
+	if affected, _ := res.RowsAffected(); affected == 0 {
+		return t.ErrNotFound
+	}
+	return nil
+}
+
+func (a *adapter) TenantKafkaOutboxMarkFailed(id t.TenantID, eventID, reason, message string, nextRetryAt, now time.Time) error {
+	tenantAdp, err := a.infrastructureAdapter(id)
+	if err != nil {
+		return err
+	}
+	if eventID == "" {
+		return t.ErrMalformed
+	}
+	if tenantAdp != a {
+		return tenantAdp.TenantKafkaOutboxMarkFailed(id, eventID, reason, message, nextRetryAt, now)
+	}
+	res, err := a.db.Exec(`UPDATE im_kafka_outbox SET status=?,attempts=attempts+1,next_retry_at=?,
+		last_reason=?,last_error=?,updated_at=? WHERE event_id=? AND status<>?`,
+		t.KafkaOutboxFailed, nextRetryAt, truncateKafkaString(reason, 64), truncateKafkaString(message, 1000),
+		now, eventID, t.KafkaOutboxSent)
+	if err != nil {
+		return err
+	}
+	if affected, _ := res.RowsAffected(); affected == 0 {
+		return t.ErrNotFound
+	}
+	return nil
+}
+
+func scanKafkaDeadLetter(rows *sqlx.Rows) (*t.KafkaDeadLetter, error) {
+	var row struct {
+		ID            int64     `db:"id"`
+		DeadID        string    `db:"dead_id"`
+		EventID       string    `db:"event_id"`
+		ParentEventID string    `db:"parent_event_id"`
+		SourceTopic   string    `db:"source_topic"`
+		ConsumerGroup string    `db:"consumer_group"`
+		EventType     string    `db:"event_type"`
+		BizType       string    `db:"biz_type"`
+		BizID         string    `db:"biz_id"`
+		Envelope      []byte    `db:"envelope"`
+		RetryCount    int       `db:"retry_count"`
+		LastReason    string    `db:"last_reason"`
+		LastError     string    `db:"last_error"`
+		Status        string    `db:"status"`
+		ReplayCount   int       `db:"replay_count"`
+		CreatedAt     time.Time `db:"created_at"`
+		UpdatedAt     time.Time `db:"updated_at"`
+	}
+	if err := rows.StructScan(&row); err != nil {
+		return nil, err
+	}
+	return &t.KafkaDeadLetter{
+		ID:            row.ID,
+		DeadID:        row.DeadID,
+		EventID:       row.EventID,
+		ParentEventID: row.ParentEventID,
+		SourceTopic:   row.SourceTopic,
+		ConsumerGroup: row.ConsumerGroup,
+		EventType:     row.EventType,
+		BizType:       row.BizType,
+		BizID:         row.BizID,
+		Envelope:      row.Envelope,
+		RetryCount:    row.RetryCount,
+		LastReason:    row.LastReason,
+		LastError:     row.LastError,
+		Status:        row.Status,
+		ReplayCount:   row.ReplayCount,
+		CreatedAt:     row.CreatedAt,
+		UpdatedAt:     row.UpdatedAt,
+	}, nil
+}
+
+func (a *adapter) kafkaConsumeGet(eventID, consumerGroup string) (*t.KafkaConsumeRecord, error) {
+	row := struct {
+		EventID            string     `db:"event_id"`
+		ConsumerGroup      string     `db:"consumer_group"`
+		SourceTopic        string     `db:"source_topic"`
+		EventType          string     `db:"event_type"`
+		BizType            string     `db:"biz_type"`
+		BizID              string     `db:"biz_id"`
+		Status             string     `db:"status"`
+		RetryCount         int        `db:"retry_count"`
+		LastReason         string     `db:"last_reason"`
+		LastError          string     `db:"last_error"`
+		ProcessingDeadline *time.Time `db:"processing_deadline"`
+		SucceededAt        *time.Time `db:"succeeded_at"`
+		DeadAt             *time.Time `db:"dead_at"`
+		CreatedAt          time.Time  `db:"created_at"`
+		UpdatedAt          time.Time  `db:"updated_at"`
+	}{}
+	err := a.db.Get(&row, `SELECT event_id,consumer_group,source_topic,event_type,biz_type,biz_id,status,retry_count,
+		last_reason,last_error,processing_deadline,succeeded_at,dead_at,created_at,updated_at
+		FROM im_kafka_consume_log WHERE event_id=? AND consumer_group=?`, eventID, consumerGroup)
+	if err == sql.ErrNoRows {
+		return nil, t.ErrNotFound
+	}
+	if err != nil {
+		return nil, err
+	}
+	return &t.KafkaConsumeRecord{EventID: row.EventID, ConsumerGroup: row.ConsumerGroup, SourceTopic: row.SourceTopic,
+		EventType: row.EventType, BizType: row.BizType, BizID: row.BizID, Status: row.Status, RetryCount: row.RetryCount,
+		LastReason: row.LastReason, LastError: row.LastError, ProcessingDeadline: row.ProcessingDeadline,
+		SucceededAt: row.SucceededAt, DeadAt: row.DeadAt, CreatedAt: row.CreatedAt, UpdatedAt: row.UpdatedAt}, nil
+}
+
+func truncateKafkaString(value string, limit int) string {
+	if limit <= 0 {
+		return ""
+	}
+	runes := []rune(value)
+	if len(runes) <= limit {
+		return value
+	}
+	return string(runes[:limit])
+}
+
 func insertBusinessEventTx(tx *sqlx.Tx, eventID, targetType, target string, actor t.Uid, eventType string, payload any) error {
 	encoded, err := json.Marshal(payload)
 	if err != nil {