feat: add kafka admin integration
This commit is contained in:
@@ -428,6 +428,31 @@ func (a *adapter) TenantGetByID(id t.TenantID) (*t.Tenant, error) {
|
||||
return a.tenantByID(id)
|
||||
}
|
||||
|
||||
func (a *adapter) TenantListActive(limit int) ([]t.Tenant, error) {
|
||||
if limit <= 0 {
|
||||
limit = 1000
|
||||
}
|
||||
ctx, cancel := a.getContext()
|
||||
if cancel != nil {
|
||||
defer cancel()
|
||||
}
|
||||
db := a.db
|
||||
if a.registryDB != nil {
|
||||
db = a.registryDB
|
||||
}
|
||||
var tenants []t.Tenant
|
||||
err := db.SelectContext(ctx, &tenants,
|
||||
`SELECT id, code, name, tenant_desc, IFNULL(db_name,'') AS db_name, IFNULL(db_dsn,'') AS db_dsn,
|
||||
state, created_at, created_by, updated_at, updated_by
|
||||
FROM im_tenant WHERE state=? ORDER BY id ASC LIMIT ?`, t.TenantStateActive, limit)
|
||||
if isUnknownColumn(err) {
|
||||
err = db.SelectContext(ctx, &tenants,
|
||||
`SELECT id, code, name, tenant_desc, state, created_at, created_by, updated_at, updated_by
|
||||
FROM im_tenant WHERE state=? ORDER BY id ASC LIMIT ?`, t.TenantStateActive, limit)
|
||||
}
|
||||
return tenants, err
|
||||
}
|
||||
|
||||
func (a *adapter) TenantAdapterFor(tenantID t.TenantID) (dbadapter.TenantBusinessAdapter, dbadapter.TenantAuthAdapter, error) {
|
||||
if !a.tenantDBRouting {
|
||||
return a, a, nil
|
||||
|
||||
@@ -29,6 +29,52 @@ func ensureIMHTTPInfrastructureTables(exec sqlExecutor) error {
|
||||
INDEX idx_im_business_events_outbox_pending(status, next_retry_at, id),
|
||||
INDEX idx_im_business_events_outbox_target(target_type, target, id)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci`,
|
||||
`CREATE TABLE IF NOT EXISTS im_kafka_consume_log (
|
||||
id BIGINT NOT NULL AUTO_INCREMENT, event_id VARCHAR(96) NOT NULL,
|
||||
consumer_group VARCHAR(128) NOT NULL, source_topic VARCHAR(128) NOT NULL,
|
||||
event_type VARCHAR(96) NOT NULL DEFAULT '', biz_type VARCHAR(64) NOT NULL DEFAULT '',
|
||||
biz_id VARCHAR(128) NOT NULL DEFAULT '', status VARCHAR(16) NOT NULL DEFAULT 'processing',
|
||||
retry_count INT NOT NULL DEFAULT 0, last_reason VARCHAR(64) NOT NULL DEFAULT '',
|
||||
last_error VARCHAR(1000) NOT NULL DEFAULT '', processing_deadline DATETIME(3),
|
||||
succeeded_at DATETIME(3), dead_at DATETIME(3),
|
||||
created_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3),
|
||||
updated_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3) ON UPDATE CURRENT_TIMESTAMP(3),
|
||||
PRIMARY KEY(id), UNIQUE INDEX uk_im_kafka_consume_log_event_group(event_id, consumer_group),
|
||||
INDEX idx_im_kafka_consume_log_status(status, processing_deadline),
|
||||
INDEX idx_im_kafka_consume_log_biz(biz_type, biz_id),
|
||||
INDEX idx_im_kafka_consume_log_topic(source_topic, id)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci`,
|
||||
`CREATE TABLE IF NOT EXISTS im_kafka_outbox (
|
||||
id BIGINT NOT NULL AUTO_INCREMENT, event_id VARCHAR(96) NOT NULL,
|
||||
topic VARCHAR(128) NOT NULL, event_type VARCHAR(96) NOT NULL,
|
||||
biz_type VARCHAR(64) NOT NULL DEFAULT '', biz_id VARCHAR(128) NOT NULL DEFAULT '',
|
||||
key_value VARCHAR(256) NOT NULL DEFAULT '', envelope JSON NOT NULL,
|
||||
status VARCHAR(16) NOT NULL DEFAULT 'pending', attempts INT NOT NULL DEFAULT 0,
|
||||
next_retry_at DATETIME(3), sent_at DATETIME(3), last_reason VARCHAR(64) NOT NULL DEFAULT '',
|
||||
last_error VARCHAR(1000) NOT NULL DEFAULT '',
|
||||
created_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3),
|
||||
updated_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3) ON UPDATE CURRENT_TIMESTAMP(3),
|
||||
PRIMARY KEY(id), UNIQUE INDEX uk_im_kafka_outbox_event(event_id),
|
||||
INDEX idx_im_kafka_outbox_pending(status, next_retry_at, id),
|
||||
INDEX idx_im_kafka_outbox_biz(biz_type, biz_id)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci`,
|
||||
`CREATE TABLE IF NOT EXISTS im_kafka_dead_letters (
|
||||
id BIGINT NOT NULL AUTO_INCREMENT, dead_id VARCHAR(96) NOT NULL,
|
||||
event_id VARCHAR(96) NOT NULL, parent_event_id VARCHAR(96) NOT NULL DEFAULT '',
|
||||
source_topic VARCHAR(128) NOT NULL, consumer_group VARCHAR(128) NOT NULL,
|
||||
event_type VARCHAR(96) NOT NULL DEFAULT '', biz_type VARCHAR(64) NOT NULL DEFAULT '',
|
||||
biz_id VARCHAR(128) NOT NULL DEFAULT '', envelope JSON NOT NULL,
|
||||
retry_count INT NOT NULL DEFAULT 0, last_reason VARCHAR(64) NOT NULL DEFAULT '',
|
||||
last_error VARCHAR(1000) NOT NULL DEFAULT '', status VARCHAR(24) NOT NULL DEFAULT 'dead',
|
||||
replay_count INT NOT NULL DEFAULT 0, resolved_by VARCHAR(64) NOT NULL DEFAULT '',
|
||||
resolved_at DATETIME(3), resolve_note VARCHAR(1000) NOT NULL DEFAULT '',
|
||||
created_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3),
|
||||
updated_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3) ON UPDATE CURRENT_TIMESTAMP(3),
|
||||
PRIMARY KEY(id), UNIQUE INDEX uk_im_kafka_dead_letters_dead(dead_id),
|
||||
INDEX idx_im_kafka_dead_letters_event(event_id),
|
||||
INDEX idx_im_kafka_dead_letters_status(status, created_at),
|
||||
INDEX idx_im_kafka_dead_letters_biz(biz_type, biz_id)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci`,
|
||||
`CREATE TABLE IF NOT EXISTS im_realname_appeals (
|
||||
id BIGINT NOT NULL, appeal_id VARCHAR(64) NOT NULL, user_id BIGINT NOT NULL,
|
||||
reason VARCHAR(500) NOT NULL DEFAULT '', attachments_json JSON,
|
||||
|
||||
@@ -12,13 +12,16 @@ func TestEnsureIMHTTPInfrastructureTablesUsesSelectedTenantDatabase(t *testing.T
|
||||
if err := ensureIMHTTPInfrastructureTables(exec); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(exec.statements) != 5 {
|
||||
t.Fatalf("created %d infrastructure tables, want 5", len(exec.statements))
|
||||
if len(exec.statements) != 8 {
|
||||
t.Fatalf("created %d infrastructure tables, want 8", len(exec.statements))
|
||||
}
|
||||
joined := strings.ToLower(strings.Join(exec.statements, "\n"))
|
||||
for _, table := range []string{
|
||||
"im_http_idempotency",
|
||||
"im_business_events_outbox",
|
||||
"im_kafka_consume_log",
|
||||
"im_kafka_outbox",
|
||||
"im_kafka_dead_letters",
|
||||
"im_realname_appeals",
|
||||
"im_conversation_settings",
|
||||
"im_topic_drafts",
|
||||
|
||||
@@ -2289,6 +2289,9 @@ func (a *adapter) TenantMessageSearch(id t.TenantID, topic string, uid t.Uid, qu
|
||||
if limit <= 0 || limit > 100 {
|
||||
limit = 50
|
||||
}
|
||||
if t.GetTopicCat(topic) == t.TopicCatGrp {
|
||||
return a.tenantGroupMessageSearch(id, topic, uid, queryText, messageType, cursor, limit)
|
||||
}
|
||||
where := []string{"m.topic=?", "m.delid=0"}
|
||||
args := []any{topic}
|
||||
if cursor != "" {
|
||||
@@ -2347,6 +2350,67 @@ func (a *adapter) TenantMessageSearch(id t.TenantID, topic string, uid t.Uid, qu
|
||||
return msgs, nextCursor, nil
|
||||
}
|
||||
|
||||
func (a *adapter) tenantGroupMessageSearch(id t.TenantID, topic string, uid t.Uid, queryText, messageType, cursor string, limit int) ([]t.Message, string, error) {
|
||||
where := []string{"m.status!='deleted'", "m.topic=?"}
|
||||
args := []any{topic}
|
||||
if cursor != "" {
|
||||
parsed, err := strconv.Atoi(cursor)
|
||||
if err != nil || parsed <= 0 {
|
||||
return nil, "", t.ErrMalformed
|
||||
}
|
||||
where = append(where, "m.seqid<?")
|
||||
args = append(args, parsed)
|
||||
}
|
||||
queryText = strings.TrimSpace(queryText)
|
||||
if queryText != "" {
|
||||
where = append(where, "CAST(m.content AS CHAR) LIKE ?")
|
||||
args = append(args, "%"+queryText+"%")
|
||||
}
|
||||
messageType = strings.TrimSpace(messageType)
|
||||
if messageType != "" && messageType != "all" {
|
||||
where = append(where, "m.msg_type=?")
|
||||
args = append(args, messageType)
|
||||
}
|
||||
ctx, cancel := a.getContext()
|
||||
if cancel != nil {
|
||||
defer cancel()
|
||||
}
|
||||
queryArgs := append([]any{store.DecodeUid(uid)}, args...)
|
||||
queryArgs = append(queryArgs, limit+1)
|
||||
rows, err := a.db.QueryxContext(ctx,
|
||||
"SELECT m.id,m.created_at AS createdat,m.updated_at AS updatedat,m.deleted_at AS deletedat,0 AS delid,m.seqid,m.topic,m.from_user AS `from`,IFNULL(m.client_msg_id,'') AS client_msg_id,m.version,m.recalled_at,m.recalled_by,m.head,m.content "+
|
||||
"FROM im_group_messages AS m LEFT JOIN im_dellog AS d "+
|
||||
"ON d.topic COLLATE utf8mb4_unicode_ci=m.topic AND m.seqid BETWEEN d.low AND d.hi-1 AND d.deletedfor=? "+
|
||||
"WHERE "+strings.Join(where, " AND ")+" AND d.deletedfor IS NULL ORDER BY m.seqid DESC LIMIT ?",
|
||||
queryArgs...)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
defer rows.Close()
|
||||
msgs := make([]t.Message, 0, limit)
|
||||
nextCursor := ""
|
||||
for rows.Next() {
|
||||
row := struct {
|
||||
t.Message
|
||||
RecalledBy sql.NullInt64 `db:"recalled_by"`
|
||||
}{}
|
||||
if err = rows.StructScan(&row); err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
msg := normalizeGroupMessage(id, row.Message, row.RecalledBy)
|
||||
msg.Head = kvMapFromJSON(msg.Head)
|
||||
if len(msgs) == limit {
|
||||
nextCursor = strconv.FormatInt(msg.SeqId, 10)
|
||||
continue
|
||||
}
|
||||
msgs = append(msgs, msg)
|
||||
}
|
||||
if err = rows.Err(); err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
return msgs, nextCursor, nil
|
||||
}
|
||||
|
||||
func (a *adapter) TenantCallCreate(id t.TenantID, record *t.CallRecord) error {
|
||||
if !a.tenantScopedDB {
|
||||
tenantAdp, err := a.routedTenantAdapter(id)
|
||||
|
||||
@@ -4,8 +4,11 @@
|
||||
package mysql
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/jmoiron/sqlx"
|
||||
@@ -225,6 +228,512 @@ func (a *adapter) TenantBusinessEventMarkDelivered(id t.TenantID, eventID string
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a *adapter) TenantKafkaConsumeBegin(id t.TenantID, record *t.KafkaConsumeRecord, deadline time.Time) (*t.KafkaConsumeRecord, bool, error) {
|
||||
tenantAdp, err := a.infrastructureAdapter(id)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
if record == nil || record.EventID == "" || record.ConsumerGroup == "" || record.SourceTopic == "" {
|
||||
return nil, false, t.ErrMalformed
|
||||
}
|
||||
if tenantAdp != a {
|
||||
return tenantAdp.TenantKafkaConsumeBegin(id, record, deadline)
|
||||
}
|
||||
now := t.TimeNow()
|
||||
_, err = a.db.Exec(`INSERT INTO im_kafka_consume_log
|
||||
(event_id,consumer_group,source_topic,event_type,biz_type,biz_id,status,processing_deadline,created_at,updated_at)
|
||||
VALUES(?,?,?,?,?,?,?,?,?,?)`, record.EventID, record.ConsumerGroup, record.SourceTopic, record.EventType,
|
||||
record.BizType, record.BizID, t.KafkaConsumeProcessing, deadline, now, now)
|
||||
if err == nil {
|
||||
record.Status = t.KafkaConsumeProcessing
|
||||
record.ProcessingDeadline = &deadline
|
||||
record.CreatedAt = now
|
||||
record.UpdatedAt = now
|
||||
return record, true, nil
|
||||
}
|
||||
if !isDupe(err) {
|
||||
return nil, false, err
|
||||
}
|
||||
existing, err := a.kafkaConsumeGet(record.EventID, record.ConsumerGroup)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
if existing.Status == t.KafkaConsumeSucceeded || existing.Status == t.KafkaConsumeDead {
|
||||
return existing, false, nil
|
||||
}
|
||||
if existing.Status == t.KafkaConsumeProcessing && existing.ProcessingDeadline != nil && existing.ProcessingDeadline.After(now) {
|
||||
return existing, false, nil
|
||||
}
|
||||
res, err := a.db.Exec(`UPDATE im_kafka_consume_log SET status=?,source_topic=?,event_type=?,biz_type=?,biz_id=?,
|
||||
processing_deadline=?,updated_at=? WHERE event_id=? AND consumer_group=? AND status<>?`,
|
||||
t.KafkaConsumeProcessing, record.SourceTopic, record.EventType, record.BizType, record.BizID,
|
||||
deadline, now, record.EventID, record.ConsumerGroup, t.KafkaConsumeSucceeded)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
if affected, _ := res.RowsAffected(); affected == 0 {
|
||||
current, getErr := a.kafkaConsumeGet(record.EventID, record.ConsumerGroup)
|
||||
return current, false, getErr
|
||||
}
|
||||
current, err := a.kafkaConsumeGet(record.EventID, record.ConsumerGroup)
|
||||
return current, err == nil, err
|
||||
}
|
||||
|
||||
func (a *adapter) TenantKafkaConsumeSucceed(id t.TenantID, eventID, consumerGroup string, now time.Time) error {
|
||||
tenantAdp, err := a.infrastructureAdapter(id)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if eventID == "" || consumerGroup == "" {
|
||||
return t.ErrMalformed
|
||||
}
|
||||
if tenantAdp != a {
|
||||
return tenantAdp.TenantKafkaConsumeSucceed(id, eventID, consumerGroup, now)
|
||||
}
|
||||
res, err := a.db.Exec(`UPDATE im_kafka_consume_log SET status=?,succeeded_at=?,processing_deadline=NULL,updated_at=?
|
||||
WHERE event_id=? AND consumer_group=?`, t.KafkaConsumeSucceeded, now, now, eventID, consumerGroup)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if affected, _ := res.RowsAffected(); affected == 0 {
|
||||
return t.ErrNotFound
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a *adapter) TenantKafkaConsumeFail(id t.TenantID, eventID, consumerGroup, reason, message string, now time.Time) (*t.KafkaConsumeRecord, error) {
|
||||
tenantAdp, err := a.infrastructureAdapter(id)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if eventID == "" || consumerGroup == "" {
|
||||
return nil, t.ErrMalformed
|
||||
}
|
||||
if tenantAdp != a {
|
||||
return tenantAdp.TenantKafkaConsumeFail(id, eventID, consumerGroup, reason, message, now)
|
||||
}
|
||||
_, err = a.db.Exec(`UPDATE im_kafka_consume_log SET status=?,retry_count=retry_count+1,last_reason=?,
|
||||
last_error=?,processing_deadline=NULL,updated_at=? WHERE event_id=? AND consumer_group=?`,
|
||||
t.KafkaConsumeFailed, truncateKafkaString(reason, 64), truncateKafkaString(message, 1000), now, eventID, consumerGroup)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return a.kafkaConsumeGet(eventID, consumerGroup)
|
||||
}
|
||||
|
||||
func (a *adapter) TenantKafkaConsumeDead(id t.TenantID, dead *t.KafkaDeadLetter, now time.Time) error {
|
||||
tenantAdp, err := a.infrastructureAdapter(id)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if dead == nil || dead.DeadID == "" || dead.EventID == "" || dead.ConsumerGroup == "" || dead.SourceTopic == "" || len(dead.Envelope) == 0 {
|
||||
return t.ErrMalformed
|
||||
}
|
||||
if tenantAdp != a {
|
||||
return tenantAdp.TenantKafkaConsumeDead(id, dead, now)
|
||||
}
|
||||
tx, err := a.db.BeginTxx(context.Background(), nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
if _, err = tx.Exec(`INSERT INTO im_kafka_dead_letters
|
||||
(dead_id,event_id,parent_event_id,source_topic,consumer_group,event_type,biz_type,biz_id,envelope,
|
||||
retry_count,last_reason,last_error,status,replay_count,created_at,updated_at)
|
||||
VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)
|
||||
ON DUPLICATE KEY UPDATE last_reason=VALUES(last_reason),last_error=VALUES(last_error),updated_at=VALUES(updated_at)`,
|
||||
dead.DeadID, dead.EventID, dead.ParentEventID, dead.SourceTopic, dead.ConsumerGroup, dead.EventType,
|
||||
dead.BizType, dead.BizID, dead.Envelope, dead.RetryCount, truncateKafkaString(dead.LastReason, 64),
|
||||
truncateKafkaString(dead.LastError, 1000), t.KafkaDeadStatusDead, dead.ReplayCount, now, now); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err = tx.Exec(`UPDATE im_kafka_consume_log SET status=?,dead_at=?,processing_deadline=NULL,last_reason=?,
|
||||
last_error=?,updated_at=? WHERE event_id=? AND consumer_group=?`,
|
||||
t.KafkaConsumeDead, now, truncateKafkaString(dead.LastReason, 64), truncateKafkaString(dead.LastError, 1000),
|
||||
now, dead.EventID, dead.ConsumerGroup); err != nil {
|
||||
return err
|
||||
}
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
func (a *adapter) TenantKafkaDeadLetterList(id t.TenantID, opts t.KafkaDeadLetterListOptions) (*t.KafkaDeadLetterPage, error) {
|
||||
tenantAdp, err := a.infrastructureAdapter(id)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if tenantAdp != a {
|
||||
return tenantAdp.TenantKafkaDeadLetterList(id, opts)
|
||||
}
|
||||
limit := opts.Limit
|
||||
if limit <= 0 || limit > 200 {
|
||||
limit = 50
|
||||
}
|
||||
where := []string{"1=1"}
|
||||
args := []any{}
|
||||
if opts.Status != "" {
|
||||
where = append(where, "status=?")
|
||||
args = append(args, opts.Status)
|
||||
}
|
||||
if opts.BizType != "" {
|
||||
where = append(where, "biz_type=?")
|
||||
args = append(args, opts.BizType)
|
||||
}
|
||||
if opts.BizID != "" {
|
||||
where = append(where, "biz_id=?")
|
||||
args = append(args, opts.BizID)
|
||||
}
|
||||
if opts.EventType != "" {
|
||||
where = append(where, "event_type=?")
|
||||
args = append(args, opts.EventType)
|
||||
}
|
||||
if opts.Cursor != "" {
|
||||
where = append(where, "id<?")
|
||||
args = append(args, opts.Cursor)
|
||||
}
|
||||
args = append(args, limit+1)
|
||||
rows, err := a.db.Queryx(`SELECT id,dead_id,event_id,parent_event_id,source_topic,consumer_group,event_type,biz_type,biz_id,
|
||||
envelope,retry_count,last_reason,last_error,status,replay_count,created_at,updated_at
|
||||
FROM im_kafka_dead_letters WHERE `+strings.Join(where, " AND ")+` ORDER BY id DESC LIMIT ?`, args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
page := &t.KafkaDeadLetterPage{Items: make([]t.KafkaDeadLetter, 0, limit)}
|
||||
for rows.Next() {
|
||||
dead, err := scanKafkaDeadLetter(rows)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(page.Items) == limit {
|
||||
page.NextCursor = strconv.FormatInt(dead.ID, 10)
|
||||
continue
|
||||
}
|
||||
page.Items = append(page.Items, *dead)
|
||||
}
|
||||
return page, rows.Err()
|
||||
}
|
||||
|
||||
func (a *adapter) TenantKafkaDeadLetterGet(id t.TenantID, deadID string) (*t.KafkaDeadLetter, error) {
|
||||
tenantAdp, err := a.infrastructureAdapter(id)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if strings.TrimSpace(deadID) == "" {
|
||||
return nil, t.ErrMalformed
|
||||
}
|
||||
if tenantAdp != a {
|
||||
return tenantAdp.TenantKafkaDeadLetterGet(id, deadID)
|
||||
}
|
||||
rows, err := a.db.Queryx(`SELECT id,dead_id,event_id,parent_event_id,source_topic,consumer_group,event_type,biz_type,biz_id,
|
||||
envelope,retry_count,last_reason,last_error,status,replay_count,created_at,updated_at
|
||||
FROM im_kafka_dead_letters WHERE dead_id=?`, strings.TrimSpace(deadID))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
if rows.Next() {
|
||||
return scanKafkaDeadLetter(rows)
|
||||
}
|
||||
if err = rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return nil, t.ErrNotFound
|
||||
}
|
||||
|
||||
func (a *adapter) TenantKafkaDeadLetterMark(id t.TenantID, deadID, status, resolvedBy, note string, now time.Time) error {
|
||||
tenantAdp, err := a.infrastructureAdapter(id)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
deadID = strings.TrimSpace(deadID)
|
||||
status = strings.TrimSpace(status)
|
||||
if deadID == "" || (status != t.KafkaDeadStatusIgnored && status != t.KafkaDeadStatusResolved && status != t.KafkaDeadStatusReplayed) {
|
||||
return t.ErrMalformed
|
||||
}
|
||||
if tenantAdp != a {
|
||||
return tenantAdp.TenantKafkaDeadLetterMark(id, deadID, status, resolvedBy, note, now)
|
||||
}
|
||||
res, err := a.db.Exec(`UPDATE im_kafka_dead_letters SET status=?,resolved_by=?,resolved_at=?,resolve_note=?,updated_at=?
|
||||
WHERE dead_id=? AND status IN (?,?,?)`, status, truncateKafkaString(resolvedBy, 64), now,
|
||||
truncateKafkaString(note, 1000), now, deadID, t.KafkaDeadStatusDead, t.KafkaDeadStatusPendingReplay, t.KafkaDeadStatusReplayed)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if affected, _ := res.RowsAffected(); affected == 0 {
|
||||
return t.ErrNotFound
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a *adapter) TenantKafkaDeadLetterMarkReplay(id t.TenantID, deadID, resolvedBy, note string, now time.Time) (*t.KafkaDeadLetter, error) {
|
||||
tenantAdp, err := a.infrastructureAdapter(id)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
deadID = strings.TrimSpace(deadID)
|
||||
if deadID == "" {
|
||||
return nil, t.ErrMalformed
|
||||
}
|
||||
if tenantAdp != a {
|
||||
return tenantAdp.TenantKafkaDeadLetterMarkReplay(id, deadID, resolvedBy, note, now)
|
||||
}
|
||||
tx, err := a.db.BeginTxx(context.Background(), nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
var row struct {
|
||||
EventID string `db:"event_id"`
|
||||
ConsumerGroup string `db:"consumer_group"`
|
||||
}
|
||||
if err = tx.Get(&row, `SELECT event_id,consumer_group FROM im_kafka_dead_letters WHERE dead_id=? FOR UPDATE`, deadID); err != nil {
|
||||
if err == sql.ErrNoRows {
|
||||
return nil, t.ErrNotFound
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
res, err := tx.Exec(`UPDATE im_kafka_dead_letters SET status=?,replay_count=replay_count+1,resolved_by=?,
|
||||
resolved_at=?,resolve_note=?,updated_at=? WHERE dead_id=? AND status IN (?,?)`,
|
||||
t.KafkaDeadStatusPendingReplay, truncateKafkaString(resolvedBy, 64), now, truncateKafkaString(note, 1000),
|
||||
now, deadID, t.KafkaDeadStatusDead, t.KafkaDeadStatusPendingReplay)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if affected, _ := res.RowsAffected(); affected == 0 {
|
||||
return nil, t.ErrNotFound
|
||||
}
|
||||
if _, err = tx.Exec(`DELETE FROM im_kafka_consume_log WHERE event_id=? AND consumer_group=?`,
|
||||
row.EventID, row.ConsumerGroup); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err = tx.Commit(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return a.TenantKafkaDeadLetterGet(id, deadID)
|
||||
}
|
||||
|
||||
func (a *adapter) TenantKafkaOutboxCreate(id t.TenantID, record *t.KafkaOutboxRecord) error {
|
||||
tenantAdp, err := a.infrastructureAdapter(id)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if record == nil || record.EventID == "" || record.Topic == "" || record.EventType == "" || len(record.Envelope) == 0 {
|
||||
return t.ErrMalformed
|
||||
}
|
||||
if tenantAdp != a {
|
||||
return tenantAdp.TenantKafkaOutboxCreate(id, record)
|
||||
}
|
||||
now := t.TimeNow()
|
||||
_, err = a.db.Exec(`INSERT INTO im_kafka_outbox
|
||||
(event_id,topic,event_type,biz_type,biz_id,key_value,envelope,status,next_retry_at,created_at,updated_at)
|
||||
VALUES(?,?,?,?,?,?,?,?,?,?,?)
|
||||
ON DUPLICATE KEY UPDATE updated_at=updated_at`,
|
||||
record.EventID, record.Topic, record.EventType, record.BizType, record.BizID, record.Key,
|
||||
record.Envelope, t.KafkaOutboxPending, record.NextRetryAt, now, now)
|
||||
return err
|
||||
}
|
||||
|
||||
func (a *adapter) TenantKafkaOutboxListPending(id t.TenantID, limit int, now time.Time) ([]t.KafkaOutboxRecord, error) {
|
||||
tenantAdp, err := a.infrastructureAdapter(id)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if limit <= 0 {
|
||||
limit = 100
|
||||
}
|
||||
if tenantAdp != a {
|
||||
return tenantAdp.TenantKafkaOutboxListPending(id, limit, now)
|
||||
}
|
||||
rows := []struct {
|
||||
ID int64 `db:"id"`
|
||||
EventID string `db:"event_id"`
|
||||
Topic string `db:"topic"`
|
||||
EventType string `db:"event_type"`
|
||||
BizType string `db:"biz_type"`
|
||||
BizID string `db:"biz_id"`
|
||||
Key string `db:"key_value"`
|
||||
Envelope []byte `db:"envelope"`
|
||||
Status string `db:"status"`
|
||||
Attempts int `db:"attempts"`
|
||||
NextRetryAt sql.NullTime `db:"next_retry_at"`
|
||||
SentAt sql.NullTime `db:"sent_at"`
|
||||
LastReason string `db:"last_reason"`
|
||||
LastError string `db:"last_error"`
|
||||
CreatedAt time.Time `db:"created_at"`
|
||||
UpdatedAt time.Time `db:"updated_at"`
|
||||
}{}
|
||||
err = a.db.Select(&rows, `SELECT id,event_id,topic,event_type,biz_type,biz_id,key_value,envelope,status,attempts,
|
||||
next_retry_at,sent_at,last_reason,last_error,created_at,updated_at
|
||||
FROM im_kafka_outbox
|
||||
WHERE status IN (?,?) AND (next_retry_at IS NULL OR next_retry_at<=?)
|
||||
ORDER BY id ASC LIMIT ?`, t.KafkaOutboxPending, t.KafkaOutboxFailed, now, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := make([]t.KafkaOutboxRecord, 0, len(rows))
|
||||
for _, row := range rows {
|
||||
rec := t.KafkaOutboxRecord{
|
||||
ID: row.ID,
|
||||
EventID: row.EventID,
|
||||
Topic: row.Topic,
|
||||
EventType: row.EventType,
|
||||
BizType: row.BizType,
|
||||
BizID: row.BizID,
|
||||
Key: row.Key,
|
||||
Envelope: row.Envelope,
|
||||
Status: row.Status,
|
||||
Attempts: row.Attempts,
|
||||
LastReason: row.LastReason,
|
||||
LastError: row.LastError,
|
||||
CreatedAt: row.CreatedAt,
|
||||
UpdatedAt: row.UpdatedAt,
|
||||
}
|
||||
if row.NextRetryAt.Valid {
|
||||
rec.NextRetryAt = &row.NextRetryAt.Time
|
||||
}
|
||||
if row.SentAt.Valid {
|
||||
rec.SentAt = &row.SentAt.Time
|
||||
}
|
||||
out = append(out, rec)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (a *adapter) TenantKafkaOutboxMarkSent(id t.TenantID, eventID string, now time.Time) error {
|
||||
tenantAdp, err := a.infrastructureAdapter(id)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if eventID == "" {
|
||||
return t.ErrMalformed
|
||||
}
|
||||
if tenantAdp != a {
|
||||
return tenantAdp.TenantKafkaOutboxMarkSent(id, eventID, now)
|
||||
}
|
||||
res, err := a.db.Exec(`UPDATE im_kafka_outbox SET status=?,sent_at=?,attempts=attempts+1,last_reason='',
|
||||
last_error='',updated_at=? WHERE event_id=? AND status<>?`,
|
||||
t.KafkaOutboxSent, now, now, eventID, t.KafkaOutboxSent)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if affected, _ := res.RowsAffected(); affected == 0 {
|
||||
return t.ErrNotFound
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a *adapter) TenantKafkaOutboxMarkFailed(id t.TenantID, eventID, reason, message string, nextRetryAt, now time.Time) error {
|
||||
tenantAdp, err := a.infrastructureAdapter(id)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if eventID == "" {
|
||||
return t.ErrMalformed
|
||||
}
|
||||
if tenantAdp != a {
|
||||
return tenantAdp.TenantKafkaOutboxMarkFailed(id, eventID, reason, message, nextRetryAt, now)
|
||||
}
|
||||
res, err := a.db.Exec(`UPDATE im_kafka_outbox SET status=?,attempts=attempts+1,next_retry_at=?,
|
||||
last_reason=?,last_error=?,updated_at=? WHERE event_id=? AND status<>?`,
|
||||
t.KafkaOutboxFailed, nextRetryAt, truncateKafkaString(reason, 64), truncateKafkaString(message, 1000),
|
||||
now, eventID, t.KafkaOutboxSent)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if affected, _ := res.RowsAffected(); affected == 0 {
|
||||
return t.ErrNotFound
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func scanKafkaDeadLetter(rows *sqlx.Rows) (*t.KafkaDeadLetter, error) {
|
||||
var row struct {
|
||||
ID int64 `db:"id"`
|
||||
DeadID string `db:"dead_id"`
|
||||
EventID string `db:"event_id"`
|
||||
ParentEventID string `db:"parent_event_id"`
|
||||
SourceTopic string `db:"source_topic"`
|
||||
ConsumerGroup string `db:"consumer_group"`
|
||||
EventType string `db:"event_type"`
|
||||
BizType string `db:"biz_type"`
|
||||
BizID string `db:"biz_id"`
|
||||
Envelope []byte `db:"envelope"`
|
||||
RetryCount int `db:"retry_count"`
|
||||
LastReason string `db:"last_reason"`
|
||||
LastError string `db:"last_error"`
|
||||
Status string `db:"status"`
|
||||
ReplayCount int `db:"replay_count"`
|
||||
CreatedAt time.Time `db:"created_at"`
|
||||
UpdatedAt time.Time `db:"updated_at"`
|
||||
}
|
||||
if err := rows.StructScan(&row); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &t.KafkaDeadLetter{
|
||||
ID: row.ID,
|
||||
DeadID: row.DeadID,
|
||||
EventID: row.EventID,
|
||||
ParentEventID: row.ParentEventID,
|
||||
SourceTopic: row.SourceTopic,
|
||||
ConsumerGroup: row.ConsumerGroup,
|
||||
EventType: row.EventType,
|
||||
BizType: row.BizType,
|
||||
BizID: row.BizID,
|
||||
Envelope: row.Envelope,
|
||||
RetryCount: row.RetryCount,
|
||||
LastReason: row.LastReason,
|
||||
LastError: row.LastError,
|
||||
Status: row.Status,
|
||||
ReplayCount: row.ReplayCount,
|
||||
CreatedAt: row.CreatedAt,
|
||||
UpdatedAt: row.UpdatedAt,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (a *adapter) kafkaConsumeGet(eventID, consumerGroup string) (*t.KafkaConsumeRecord, error) {
|
||||
row := struct {
|
||||
EventID string `db:"event_id"`
|
||||
ConsumerGroup string `db:"consumer_group"`
|
||||
SourceTopic string `db:"source_topic"`
|
||||
EventType string `db:"event_type"`
|
||||
BizType string `db:"biz_type"`
|
||||
BizID string `db:"biz_id"`
|
||||
Status string `db:"status"`
|
||||
RetryCount int `db:"retry_count"`
|
||||
LastReason string `db:"last_reason"`
|
||||
LastError string `db:"last_error"`
|
||||
ProcessingDeadline *time.Time `db:"processing_deadline"`
|
||||
SucceededAt *time.Time `db:"succeeded_at"`
|
||||
DeadAt *time.Time `db:"dead_at"`
|
||||
CreatedAt time.Time `db:"created_at"`
|
||||
UpdatedAt time.Time `db:"updated_at"`
|
||||
}{}
|
||||
err := a.db.Get(&row, `SELECT event_id,consumer_group,source_topic,event_type,biz_type,biz_id,status,retry_count,
|
||||
last_reason,last_error,processing_deadline,succeeded_at,dead_at,created_at,updated_at
|
||||
FROM im_kafka_consume_log WHERE event_id=? AND consumer_group=?`, eventID, consumerGroup)
|
||||
if err == sql.ErrNoRows {
|
||||
return nil, t.ErrNotFound
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &t.KafkaConsumeRecord{EventID: row.EventID, ConsumerGroup: row.ConsumerGroup, SourceTopic: row.SourceTopic,
|
||||
EventType: row.EventType, BizType: row.BizType, BizID: row.BizID, Status: row.Status, RetryCount: row.RetryCount,
|
||||
LastReason: row.LastReason, LastError: row.LastError, ProcessingDeadline: row.ProcessingDeadline,
|
||||
SucceededAt: row.SucceededAt, DeadAt: row.DeadAt, CreatedAt: row.CreatedAt, UpdatedAt: row.UpdatedAt}, nil
|
||||
}
|
||||
|
||||
func truncateKafkaString(value string, limit int) string {
|
||||
if limit <= 0 {
|
||||
return ""
|
||||
}
|
||||
runes := []rune(value)
|
||||
if len(runes) <= limit {
|
||||
return value
|
||||
}
|
||||
return string(runes[:limit])
|
||||
}
|
||||
|
||||
func insertBusinessEventTx(tx *sqlx.Tx, eventID, targetType, target string, actor t.Uid, eventType string, payload any) error {
|
||||
encoded, err := json.Marshal(payload)
|
||||
if err != nil {
|
||||
|
||||
@@ -0,0 +1,126 @@
|
||||
//go:build mysql
|
||||
// +build mysql
|
||||
|
||||
package mysql
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/jmoiron/sqlx"
|
||||
t "github.com/tinode/chat/server/store/types"
|
||||
)
|
||||
|
||||
func (a *adapter) TenantInternalAdminQuery(id t.TenantID, query string, countQuery string, args []any, page, perPage int) (*t.InternalAdminQueryResult, error) {
|
||||
tenantAdp, err := a.routedTenantAdapter(id)
|
||||
if err != nil {
|
||||
return nil, t.ErrMalformed
|
||||
}
|
||||
if tenantAdp != a {
|
||||
return tenantAdp.TenantInternalAdminQuery(id, query, countQuery, args, page, perPage)
|
||||
}
|
||||
if strings.TrimSpace(query) == "" || strings.TrimSpace(countQuery) == "" {
|
||||
return nil, t.ErrMalformed
|
||||
}
|
||||
if page <= 0 {
|
||||
page = 1
|
||||
}
|
||||
if perPage <= 0 || perPage > 200 {
|
||||
perPage = 20
|
||||
}
|
||||
offset := (page - 1) * perPage
|
||||
ctx, cancel := a.getContext()
|
||||
if cancel != nil {
|
||||
defer cancel()
|
||||
}
|
||||
var total int64
|
||||
if err = a.db.GetContext(ctx, &total, countQuery, args...); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
queryArgs := append(append([]any{}, args...), perPage, offset)
|
||||
rows, err := a.db.QueryxContext(ctx, query+" LIMIT ? OFFSET ?", queryArgs...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
items, err := scanInternalAdminRows(rows)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &t.InternalAdminQueryResult{Items: items, Total: total, Page: page, PerPage: perPage}, nil
|
||||
}
|
||||
|
||||
func (a *adapter) TenantInternalAdminQueryOne(id t.TenantID, query string, args []any) (map[string]any, error) {
|
||||
tenantAdp, err := a.routedTenantAdapter(id)
|
||||
if err != nil {
|
||||
return nil, t.ErrMalformed
|
||||
}
|
||||
if tenantAdp != a {
|
||||
return tenantAdp.TenantInternalAdminQueryOne(id, query, args)
|
||||
}
|
||||
if strings.TrimSpace(query) == "" {
|
||||
return nil, t.ErrMalformed
|
||||
}
|
||||
ctx, cancel := a.getContext()
|
||||
if cancel != nil {
|
||||
defer cancel()
|
||||
}
|
||||
rows, err := a.db.QueryxContext(ctx, query+" LIMIT 1", args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
items, err := scanInternalAdminRows(rows)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(items) == 0 {
|
||||
return nil, t.ErrNotFound
|
||||
}
|
||||
return items[0], nil
|
||||
}
|
||||
|
||||
func scanInternalAdminRows(rows *sqlx.Rows) ([]map[string]any, error) {
|
||||
items := []map[string]any{}
|
||||
for rows.Next() {
|
||||
row := map[string]any{}
|
||||
if err := rows.MapScan(row); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for key, value := range row {
|
||||
row[key] = normalizeInternalAdminValue(value)
|
||||
}
|
||||
items = append(items, row)
|
||||
}
|
||||
return items, rows.Err()
|
||||
}
|
||||
|
||||
func normalizeInternalAdminValue(value any) any {
|
||||
switch v := value.(type) {
|
||||
case nil:
|
||||
return nil
|
||||
case []byte:
|
||||
raw := strings.TrimSpace(string(v))
|
||||
if raw == "" {
|
||||
return ""
|
||||
}
|
||||
if (strings.HasPrefix(raw, "{") && strings.HasSuffix(raw, "}")) || (strings.HasPrefix(raw, "[") && strings.HasSuffix(raw, "]")) {
|
||||
var parsed any
|
||||
if json.Unmarshal([]byte(raw), &parsed) == nil {
|
||||
return parsed
|
||||
}
|
||||
}
|
||||
return raw
|
||||
case time.Time:
|
||||
return v.UTC().Format(time.RFC3339Nano)
|
||||
case sql.NullTime:
|
||||
if v.Valid {
|
||||
return v.Time.UTC().Format(time.RFC3339Nano)
|
||||
}
|
||||
return nil
|
||||
default:
|
||||
return value
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user