pro/im_server
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add kafka admin integration

Browse Source
This commit is contained in:
richjohnrambo
2026-08-28 00:05:54 +08:00
parent b2e5097708
commit 5cc6eebc1a
48 changed files with 8065 additions and 55 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/db/adapter.go
+18
View File
@@ -214,6 +214,7 @@ type Adapter interface {
type TenantAdapter interface {
TenantGetByCode(code string) (*t.Tenant, error)
TenantGetByID(id t.TenantID) (*t.Tenant, error)
TenantListActive(limit int) ([]t.Tenant, error)
}
// RegistrationConfigAdapter reads global registration policy from the control database.
@@ -234,6 +235,11 @@ type TenantEntryAdapter interface {
TenantEntryFeedbackCreate(*t.TenantEntryFeedback) (*t.TenantEntryFeedback, error)
}
type TenantInternalAdminQueryAdapter interface {
TenantInternalAdminQuery(tenantID t.TenantID, query string, countQuery string, args []any, page, perPage int) (*t.InternalAdminQueryResult, error)
TenantInternalAdminQueryOne(tenantID t.TenantID, query string, args []any) (map[string]any, error)
}
// TenantRoutedAdapter can return a tenant-specific adapter, e.g. for the
// "one tenant, one database" deployment model.
type TenantRoutedAdapter interface {
@@ -249,6 +255,18 @@ type TenantHTTPInfrastructureAdapter interface {
TenantBusinessEventCreate(t.TenantID, *t.BusinessEvent) error
TenantBusinessEventListPending(t.TenantID, int, time.Time) ([]t.BusinessEvent, error)
TenantBusinessEventMarkDelivered(t.TenantID, string, time.Time) error
TenantKafkaConsumeBegin(t.TenantID, *t.KafkaConsumeRecord, time.Time) (*t.KafkaConsumeRecord, bool, error)
TenantKafkaConsumeSucceed(t.TenantID, string, string, time.Time) error
TenantKafkaConsumeFail(t.TenantID, string, string, string, string, time.Time) (*t.KafkaConsumeRecord, error)
TenantKafkaConsumeDead(t.TenantID, *t.KafkaDeadLetter, time.Time) error
TenantKafkaDeadLetterList(t.TenantID, t.KafkaDeadLetterListOptions) (*t.KafkaDeadLetterPage, error)
TenantKafkaDeadLetterGet(t.TenantID, string) (*t.KafkaDeadLetter, error)
TenantKafkaDeadLetterMark(t.TenantID, string, string, string, string, time.Time) error
TenantKafkaDeadLetterMarkReplay(t.TenantID, string, string, string, time.Time) (*t.KafkaDeadLetter, error)
TenantKafkaOutboxCreate(t.TenantID, *t.KafkaOutboxRecord) error
TenantKafkaOutboxListPending(t.TenantID, int, time.Time) ([]t.KafkaOutboxRecord, error)
TenantKafkaOutboxMarkSent(t.TenantID, string, time.Time) error
TenantKafkaOutboxMarkFailed(t.TenantID, string, string, string, time.Time, time.Time) error
}
// TenantRegistrationInviteAdapter stores invitation definitions in the
server/db/mysql/adapter.go
+25
View File
@@ -428,6 +428,31 @@ func (a *adapter) TenantGetByID(id t.TenantID) (*t.Tenant, error) {
return a.tenantByID(id)
}
func (a *adapter) TenantListActive(limit int) ([]t.Tenant, error) {
if limit <= 0 {
limit = 1000
}
ctx, cancel := a.getContext()
if cancel != nil {
defer cancel()
}
db := a.db
if a.registryDB != nil {
db = a.registryDB
}
var tenants []t.Tenant
err := db.SelectContext(ctx, &tenants,
`SELECT id, code, name, tenant_desc, IFNULL(db_name,'') AS db_name, IFNULL(db_dsn,'') AS db_dsn,
state, created_at, created_by, updated_at, updated_by
FROM im_tenant WHERE state=? ORDER BY id ASC LIMIT ?`, t.TenantStateActive, limit)
if isUnknownColumn(err) {
err = db.SelectContext(ctx, &tenants,
`SELECT id, code, name, tenant_desc, state, created_at, created_by, updated_at, updated_by
FROM im_tenant WHERE state=? ORDER BY id ASC LIMIT ?`, t.TenantStateActive, limit)
}
return tenants, err
}
func (a *adapter) TenantAdapterFor(tenantID t.TenantID) (dbadapter.TenantBusinessAdapter, dbadapter.TenantAuthAdapter, error) {
if !a.tenantDBRouting {
return a, a, nil
server/db/mysql/infrastructure_schema.go
+46
View File
@@ -29,6 +29,52 @@ func ensureIMHTTPInfrastructureTables(exec sqlExecutor) error {
INDEX idx_im_business_events_outbox_pending(status, next_retry_at, id),
INDEX idx_im_business_events_outbox_target(target_type, target, id)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci`,
`CREATE TABLE IF NOT EXISTS im_kafka_consume_log (
id BIGINT NOT NULL AUTO_INCREMENT, event_id VARCHAR(96) NOT NULL,
consumer_group VARCHAR(128) NOT NULL, source_topic VARCHAR(128) NOT NULL,
event_type VARCHAR(96) NOT NULL DEFAULT '', biz_type VARCHAR(64) NOT NULL DEFAULT '',
biz_id VARCHAR(128) NOT NULL DEFAULT '', status VARCHAR(16) NOT NULL DEFAULT 'processing',
retry_count INT NOT NULL DEFAULT 0, last_reason VARCHAR(64) NOT NULL DEFAULT '',
last_error VARCHAR(1000) NOT NULL DEFAULT '', processing_deadline DATETIME(3),
succeeded_at DATETIME(3), dead_at DATETIME(3),
created_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3),
updated_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3) ON UPDATE CURRENT_TIMESTAMP(3),
PRIMARY KEY(id), UNIQUE INDEX uk_im_kafka_consume_log_event_group(event_id, consumer_group),
INDEX idx_im_kafka_consume_log_status(status, processing_deadline),
INDEX idx_im_kafka_consume_log_biz(biz_type, biz_id),
INDEX idx_im_kafka_consume_log_topic(source_topic, id)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci`,
`CREATE TABLE IF NOT EXISTS im_kafka_outbox (
id BIGINT NOT NULL AUTO_INCREMENT, event_id VARCHAR(96) NOT NULL,
topic VARCHAR(128) NOT NULL, event_type VARCHAR(96) NOT NULL,
biz_type VARCHAR(64) NOT NULL DEFAULT '', biz_id VARCHAR(128) NOT NULL DEFAULT '',
key_value VARCHAR(256) NOT NULL DEFAULT '', envelope JSON NOT NULL,
status VARCHAR(16) NOT NULL DEFAULT 'pending', attempts INT NOT NULL DEFAULT 0,
next_retry_at DATETIME(3), sent_at DATETIME(3), last_reason VARCHAR(64) NOT NULL DEFAULT '',
last_error VARCHAR(1000) NOT NULL DEFAULT '',
created_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3),
updated_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3) ON UPDATE CURRENT_TIMESTAMP(3),
PRIMARY KEY(id), UNIQUE INDEX uk_im_kafka_outbox_event(event_id),
INDEX idx_im_kafka_outbox_pending(status, next_retry_at, id),
INDEX idx_im_kafka_outbox_biz(biz_type, biz_id)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci`,
`CREATE TABLE IF NOT EXISTS im_kafka_dead_letters (
id BIGINT NOT NULL AUTO_INCREMENT, dead_id VARCHAR(96) NOT NULL,
event_id VARCHAR(96) NOT NULL, parent_event_id VARCHAR(96) NOT NULL DEFAULT '',
source_topic VARCHAR(128) NOT NULL, consumer_group VARCHAR(128) NOT NULL,
event_type VARCHAR(96) NOT NULL DEFAULT '', biz_type VARCHAR(64) NOT NULL DEFAULT '',
biz_id VARCHAR(128) NOT NULL DEFAULT '', envelope JSON NOT NULL,
retry_count INT NOT NULL DEFAULT 0, last_reason VARCHAR(64) NOT NULL DEFAULT '',
last_error VARCHAR(1000) NOT NULL DEFAULT '', status VARCHAR(24) NOT NULL DEFAULT 'dead',
replay_count INT NOT NULL DEFAULT 0, resolved_by VARCHAR(64) NOT NULL DEFAULT '',
resolved_at DATETIME(3), resolve_note VARCHAR(1000) NOT NULL DEFAULT '',
created_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3),
updated_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3) ON UPDATE CURRENT_TIMESTAMP(3),
PRIMARY KEY(id), UNIQUE INDEX uk_im_kafka_dead_letters_dead(dead_id),
INDEX idx_im_kafka_dead_letters_event(event_id),
INDEX idx_im_kafka_dead_letters_status(status, created_at),
INDEX idx_im_kafka_dead_letters_biz(biz_type, biz_id)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci`,
`CREATE TABLE IF NOT EXISTS im_realname_appeals (
id BIGINT NOT NULL, appeal_id VARCHAR(64) NOT NULL, user_id BIGINT NOT NULL,
reason VARCHAR(500) NOT NULL DEFAULT '', attachments_json JSON,
server/db/mysql/infrastructure_schema_test.go
+5 -2
View File
@@ -12,13 +12,16 @@ func TestEnsureIMHTTPInfrastructureTablesUsesSelectedTenantDatabase(t *testing.T
if err := ensureIMHTTPInfrastructureTables(exec); err != nil {
t.Fatal(err)
}
if len(exec.statements) != 5 {
t.Fatalf("created %d infrastructure tables, want 5", len(exec.statements))
if len(exec.statements) != 8 {
t.Fatalf("created %d infrastructure tables, want 8", len(exec.statements))
}
joined := strings.ToLower(strings.Join(exec.statements, "\n"))
for _, table := range []string{
"im_http_idempotency",
"im_business_events_outbox",
"im_kafka_consume_log",
"im_kafka_outbox",
"im_kafka_dead_letters",
"im_realname_appeals",
"im_conversation_settings",
"im_topic_drafts",
server/db/mysql/tenant_adapter.go
+64
View File
@@ -2289,6 +2289,9 @@ func (a *adapter) TenantMessageSearch(id t.TenantID, topic string, uid t.Uid, qu
if limit <= 0 || limit > 100 {
limit = 50
}
if t.GetTopicCat(topic) == t.TopicCatGrp {
return a.tenantGroupMessageSearch(id, topic, uid, queryText, messageType, cursor, limit)
}
where := []string{"m.topic=?", "m.delid=0"}
args := []any{topic}
if cursor != "" {
@@ -2347,6 +2350,67 @@ func (a *adapter) TenantMessageSearch(id t.TenantID, topic string, uid t.Uid, qu
return msgs, nextCursor, nil
}
func (a *adapter) tenantGroupMessageSearch(id t.TenantID, topic string, uid t.Uid, queryText, messageType, cursor string, limit int) ([]t.Message, string, error) {
where := []string{"m.status!='deleted'", "m.topic=?"}
args := []any{topic}
if cursor != "" {
parsed, err := strconv.Atoi(cursor)
if err != nil || parsed <= 0 {
return nil, "", t.ErrMalformed
}
where = append(where, "m.seqid<?")
args = append(args, parsed)
}
queryText = strings.TrimSpace(queryText)
if queryText != "" {
where = append(where, "CAST(m.content AS CHAR) LIKE ?")
args = append(args, "%"+queryText+"%")
}
messageType = strings.TrimSpace(messageType)
if messageType != "" && messageType != "all" {
where = append(where, "m.msg_type=?")
args = append(args, messageType)
}
ctx, cancel := a.getContext()
if cancel != nil {
defer cancel()
}
queryArgs := append([]any{store.DecodeUid(uid)}, args...)
queryArgs = append(queryArgs, limit+1)
rows, err := a.db.QueryxContext(ctx,
"SELECT m.id,m.created_at AS createdat,m.updated_at AS updatedat,m.deleted_at AS deletedat,0 AS delid,m.seqid,m.topic,m.from_user AS `from`,IFNULL(m.client_msg_id,'') AS client_msg_id,m.version,m.recalled_at,m.recalled_by,m.head,m.content "+
"FROM im_group_messages AS m LEFT JOIN im_dellog AS d "+
"ON d.topic COLLATE utf8mb4_unicode_ci=m.topic AND m.seqid BETWEEN d.low AND d.hi-1 AND d.deletedfor=? "+
"WHERE "+strings.Join(where, " AND ")+" AND d.deletedfor IS NULL ORDER BY m.seqid DESC LIMIT ?",
queryArgs...)
if err != nil {
return nil, "", err
}
defer rows.Close()
msgs := make([]t.Message, 0, limit)
nextCursor := ""
for rows.Next() {
row := struct {
t.Message
RecalledBy sql.NullInt64 `db:"recalled_by"`
}{}
if err = rows.StructScan(&row); err != nil {
return nil, "", err
}
msg := normalizeGroupMessage(id, row.Message, row.RecalledBy)
msg.Head = kvMapFromJSON(msg.Head)
if len(msgs) == limit {
nextCursor = strconv.FormatInt(msg.SeqId, 10)
continue
}
msgs = append(msgs, msg)
}
if err = rows.Err(); err != nil {
return nil, "", err
}
return msgs, nextCursor, nil
}
func (a *adapter) TenantCallCreate(id t.TenantID, record *t.CallRecord) error {
if !a.tenantScopedDB {
tenantAdp, err := a.routedTenantAdapter(id)
server/db/mysql/tenant_infrastructure.go
+509
View File
@@ -4,8 +4,11 @@
package mysql
import (
"context"
"database/sql"
"encoding/json"
"strconv"
"strings"
"time"
"github.com/jmoiron/sqlx"
@@ -225,6 +228,512 @@ func (a *adapter) TenantBusinessEventMarkDelivered(id t.TenantID, eventID string
return nil
}
func (a *adapter) TenantKafkaConsumeBegin(id t.TenantID, record *t.KafkaConsumeRecord, deadline time.Time) (*t.KafkaConsumeRecord, bool, error) {
tenantAdp, err := a.infrastructureAdapter(id)
if err != nil {
return nil, false, err
}
if record == nil || record.EventID == "" || record.ConsumerGroup == "" || record.SourceTopic == "" {
return nil, false, t.ErrMalformed
}
if tenantAdp != a {
return tenantAdp.TenantKafkaConsumeBegin(id, record, deadline)
}
now := t.TimeNow()
_, err = a.db.Exec(`INSERT INTO im_kafka_consume_log
(event_id,consumer_group,source_topic,event_type,biz_type,biz_id,status,processing_deadline,created_at,updated_at)
VALUES(?,?,?,?,?,?,?,?,?,?)`, record.EventID, record.ConsumerGroup, record.SourceTopic, record.EventType,
record.BizType, record.BizID, t.KafkaConsumeProcessing, deadline, now, now)
if err == nil {
record.Status = t.KafkaConsumeProcessing
record.ProcessingDeadline = &deadline
record.CreatedAt = now
record.UpdatedAt = now
return record, true, nil
}
if !isDupe(err) {
return nil, false, err
}
existing, err := a.kafkaConsumeGet(record.EventID, record.ConsumerGroup)
if err != nil {
return nil, false, err
}
if existing.Status == t.KafkaConsumeSucceeded || existing.Status == t.KafkaConsumeDead {
return existing, false, nil
}
if existing.Status == t.KafkaConsumeProcessing && existing.ProcessingDeadline != nil && existing.ProcessingDeadline.After(now) {
return existing, false, nil
}
res, err := a.db.Exec(`UPDATE im_kafka_consume_log SET status=?,source_topic=?,event_type=?,biz_type=?,biz_id=?,
processing_deadline=?,updated_at=? WHERE event_id=? AND consumer_group=? AND status<>?`,
t.KafkaConsumeProcessing, record.SourceTopic, record.EventType, record.BizType, record.BizID,
deadline, now, record.EventID, record.ConsumerGroup, t.KafkaConsumeSucceeded)
if err != nil {
return nil, false, err
}
if affected, _ := res.RowsAffected(); affected == 0 {
current, getErr := a.kafkaConsumeGet(record.EventID, record.ConsumerGroup)
return current, false, getErr
}
current, err := a.kafkaConsumeGet(record.EventID, record.ConsumerGroup)
return current, err == nil, err
}
func (a *adapter) TenantKafkaConsumeSucceed(id t.TenantID, eventID, consumerGroup string, now time.Time) error {
tenantAdp, err := a.infrastructureAdapter(id)
if err != nil {
return err
}
if eventID == "" || consumerGroup == "" {
return t.ErrMalformed
}
if tenantAdp != a {
return tenantAdp.TenantKafkaConsumeSucceed(id, eventID, consumerGroup, now)
}
res, err := a.db.Exec(`UPDATE im_kafka_consume_log SET status=?,succeeded_at=?,processing_deadline=NULL,updated_at=?
WHERE event_id=? AND consumer_group=?`, t.KafkaConsumeSucceeded, now, now, eventID, consumerGroup)
if err != nil {
return err
}
if affected, _ := res.RowsAffected(); affected == 0 {
return t.ErrNotFound
}
return nil
}
func (a *adapter) TenantKafkaConsumeFail(id t.TenantID, eventID, consumerGroup, reason, message string, now time.Time) (*t.KafkaConsumeRecord, error) {
tenantAdp, err := a.infrastructureAdapter(id)
if err != nil {
return nil, err
}
if eventID == "" || consumerGroup == "" {
return nil, t.ErrMalformed
}
if tenantAdp != a {
return tenantAdp.TenantKafkaConsumeFail(id, eventID, consumerGroup, reason, message, now)
}
_, err = a.db.Exec(`UPDATE im_kafka_consume_log SET status=?,retry_count=retry_count+1,last_reason=?,
last_error=?,processing_deadline=NULL,updated_at=? WHERE event_id=? AND consumer_group=?`,
t.KafkaConsumeFailed, truncateKafkaString(reason, 64), truncateKafkaString(message, 1000), now, eventID, consumerGroup)
if err != nil {
return nil, err
}
return a.kafkaConsumeGet(eventID, consumerGroup)
}
func (a *adapter) TenantKafkaConsumeDead(id t.TenantID, dead *t.KafkaDeadLetter, now time.Time) error {
tenantAdp, err := a.infrastructureAdapter(id)
if err != nil {
return err
}
if dead == nil || dead.DeadID == "" || dead.EventID == "" || dead.ConsumerGroup == "" || dead.SourceTopic == "" || len(dead.Envelope) == 0 {
return t.ErrMalformed
}
if tenantAdp != a {
return tenantAdp.TenantKafkaConsumeDead(id, dead, now)
}
tx, err := a.db.BeginTxx(context.Background(), nil)
if err != nil {
return err
}
defer tx.Rollback()
if _, err = tx.Exec(`INSERT INTO im_kafka_dead_letters
(dead_id,event_id,parent_event_id,source_topic,consumer_group,event_type,biz_type,biz_id,envelope,
retry_count,last_reason,last_error,status,replay_count,created_at,updated_at)
VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)
ON DUPLICATE KEY UPDATE last_reason=VALUES(last_reason),last_error=VALUES(last_error),updated_at=VALUES(updated_at)`,
dead.DeadID, dead.EventID, dead.ParentEventID, dead.SourceTopic, dead.ConsumerGroup, dead.EventType,
dead.BizType, dead.BizID, dead.Envelope, dead.RetryCount, truncateKafkaString(dead.LastReason, 64),
truncateKafkaString(dead.LastError, 1000), t.KafkaDeadStatusDead, dead.ReplayCount, now, now); err != nil {
return err
}
if _, err = tx.Exec(`UPDATE im_kafka_consume_log SET status=?,dead_at=?,processing_deadline=NULL,last_reason=?,
last_error=?,updated_at=? WHERE event_id=? AND consumer_group=?`,
t.KafkaConsumeDead, now, truncateKafkaString(dead.LastReason, 64), truncateKafkaString(dead.LastError, 1000),
now, dead.EventID, dead.ConsumerGroup); err != nil {
return err
}
return tx.Commit()
}
func (a *adapter) TenantKafkaDeadLetterList(id t.TenantID, opts t.KafkaDeadLetterListOptions) (*t.KafkaDeadLetterPage, error) {
tenantAdp, err := a.infrastructureAdapter(id)
if err != nil {
return nil, err
}
if tenantAdp != a {
return tenantAdp.TenantKafkaDeadLetterList(id, opts)
}
limit := opts.Limit
if limit <= 0 || limit > 200 {
limit = 50
}
where := []string{"1=1"}
args := []any{}
if opts.Status != "" {
where = append(where, "status=?")
args = append(args, opts.Status)
}
if opts.BizType != "" {
where = append(where, "biz_type=?")
args = append(args, opts.BizType)
}
if opts.BizID != "" {
where = append(where, "biz_id=?")
args = append(args, opts.BizID)
}
if opts.EventType != "" {
where = append(where, "event_type=?")
args = append(args, opts.EventType)
}
if opts.Cursor != "" {
where = append(where, "id<?")
args = append(args, opts.Cursor)
}
args = append(args, limit+1)
rows, err := a.db.Queryx(`SELECT id,dead_id,event_id,parent_event_id,source_topic,consumer_group,event_type,biz_type,biz_id,
envelope,retry_count,last_reason,last_error,status,replay_count,created_at,updated_at
FROM im_kafka_dead_letters WHERE `+strings.Join(where, " AND ")+` ORDER BY id DESC LIMIT ?`, args...)
if err != nil {
return nil, err
}
defer rows.Close()
page := &t.KafkaDeadLetterPage{Items: make([]t.KafkaDeadLetter, 0, limit)}
for rows.Next() {
dead, err := scanKafkaDeadLetter(rows)
if err != nil {
return nil, err
}
if len(page.Items) == limit {
page.NextCursor = strconv.FormatInt(dead.ID, 10)
continue
}
page.Items = append(page.Items, *dead)
}
return page, rows.Err()
}
func (a *adapter) TenantKafkaDeadLetterGet(id t.TenantID, deadID string) (*t.KafkaDeadLetter, error) {
tenantAdp, err := a.infrastructureAdapter(id)
if err != nil {
return nil, err
}
if strings.TrimSpace(deadID) == "" {
return nil, t.ErrMalformed
}
if tenantAdp != a {
return tenantAdp.TenantKafkaDeadLetterGet(id, deadID)
}
rows, err := a.db.Queryx(`SELECT id,dead_id,event_id,parent_event_id,source_topic,consumer_group,event_type,biz_type,biz_id,
envelope,retry_count,last_reason,last_error,status,replay_count,created_at,updated_at
FROM im_kafka_dead_letters WHERE dead_id=?`, strings.TrimSpace(deadID))
if err != nil {
return nil, err
}
defer rows.Close()
if rows.Next() {
return scanKafkaDeadLetter(rows)
}
if err = rows.Err(); err != nil {
return nil, err
}
return nil, t.ErrNotFound
}
func (a *adapter) TenantKafkaDeadLetterMark(id t.TenantID, deadID, status, resolvedBy, note string, now time.Time) error {
tenantAdp, err := a.infrastructureAdapter(id)
if err != nil {
return err
}
deadID = strings.TrimSpace(deadID)
status = strings.TrimSpace(status)
if deadID == "" || (status != t.KafkaDeadStatusIgnored && status != t.KafkaDeadStatusResolved && status != t.KafkaDeadStatusReplayed) {
return t.ErrMalformed
}
if tenantAdp != a {
return tenantAdp.TenantKafkaDeadLetterMark(id, deadID, status, resolvedBy, note, now)
}
res, err := a.db.Exec(`UPDATE im_kafka_dead_letters SET status=?,resolved_by=?,resolved_at=?,resolve_note=?,updated_at=?
WHERE dead_id=? AND status IN (?,?,?)`, status, truncateKafkaString(resolvedBy, 64), now,
truncateKafkaString(note, 1000), now, deadID, t.KafkaDeadStatusDead, t.KafkaDeadStatusPendingReplay, t.KafkaDeadStatusReplayed)
if err != nil {
return err
}
if affected, _ := res.RowsAffected(); affected == 0 {
return t.ErrNotFound
}
return nil
}
func (a *adapter) TenantKafkaDeadLetterMarkReplay(id t.TenantID, deadID, resolvedBy, note string, now time.Time) (*t.KafkaDeadLetter, error) {
tenantAdp, err := a.infrastructureAdapter(id)
if err != nil {
return nil, err
}
deadID = strings.TrimSpace(deadID)
if deadID == "" {
return nil, t.ErrMalformed
}
if tenantAdp != a {
return tenantAdp.TenantKafkaDeadLetterMarkReplay(id, deadID, resolvedBy, note, now)
}
tx, err := a.db.BeginTxx(context.Background(), nil)
if err != nil {
return nil, err
}
defer tx.Rollback()
var row struct {
EventID string `db:"event_id"`
ConsumerGroup string `db:"consumer_group"`
}
if err = tx.Get(&row, `SELECT event_id,consumer_group FROM im_kafka_dead_letters WHERE dead_id=? FOR UPDATE`, deadID); err != nil {
if err == sql.ErrNoRows {
return nil, t.ErrNotFound
}
return nil, err
}
res, err := tx.Exec(`UPDATE im_kafka_dead_letters SET status=?,replay_count=replay_count+1,resolved_by=?,
resolved_at=?,resolve_note=?,updated_at=? WHERE dead_id=? AND status IN (?,?)`,
t.KafkaDeadStatusPendingReplay, truncateKafkaString(resolvedBy, 64), now, truncateKafkaString(note, 1000),
now, deadID, t.KafkaDeadStatusDead, t.KafkaDeadStatusPendingReplay)
if err != nil {
return nil, err
}
if affected, _ := res.RowsAffected(); affected == 0 {
return nil, t.ErrNotFound
}
if _, err = tx.Exec(`DELETE FROM im_kafka_consume_log WHERE event_id=? AND consumer_group=?`,
row.EventID, row.ConsumerGroup); err != nil {
return nil, err
}
if err = tx.Commit(); err != nil {
return nil, err
}
return a.TenantKafkaDeadLetterGet(id, deadID)
}
func (a *adapter) TenantKafkaOutboxCreate(id t.TenantID, record *t.KafkaOutboxRecord) error {
tenantAdp, err := a.infrastructureAdapter(id)
if err != nil {
return err
}
if record == nil || record.EventID == "" || record.Topic == "" || record.EventType == "" || len(record.Envelope) == 0 {
return t.ErrMalformed
}
if tenantAdp != a {
return tenantAdp.TenantKafkaOutboxCreate(id, record)
}
now := t.TimeNow()
_, err = a.db.Exec(`INSERT INTO im_kafka_outbox
(event_id,topic,event_type,biz_type,biz_id,key_value,envelope,status,next_retry_at,created_at,updated_at)
VALUES(?,?,?,?,?,?,?,?,?,?,?)
ON DUPLICATE KEY UPDATE updated_at=updated_at`,
record.EventID, record.Topic, record.EventType, record.BizType, record.BizID, record.Key,
record.Envelope, t.KafkaOutboxPending, record.NextRetryAt, now, now)
return err
}
func (a *adapter) TenantKafkaOutboxListPending(id t.TenantID, limit int, now time.Time) ([]t.KafkaOutboxRecord, error) {
tenantAdp, err := a.infrastructureAdapter(id)
if err != nil {
return nil, err
}
if limit <= 0 {
limit = 100
}
if tenantAdp != a {
return tenantAdp.TenantKafkaOutboxListPending(id, limit, now)
}
rows := []struct {
ID int64 `db:"id"`
EventID string `db:"event_id"`
Topic string `db:"topic"`
EventType string `db:"event_type"`
BizType string `db:"biz_type"`
BizID string `db:"biz_id"`
Key string `db:"key_value"`
Envelope []byte `db:"envelope"`
Status string `db:"status"`
Attempts int `db:"attempts"`
NextRetryAt sql.NullTime `db:"next_retry_at"`
SentAt sql.NullTime `db:"sent_at"`
LastReason string `db:"last_reason"`
LastError string `db:"last_error"`
CreatedAt time.Time `db:"created_at"`
UpdatedAt time.Time `db:"updated_at"`
}{}
err = a.db.Select(&rows, `SELECT id,event_id,topic,event_type,biz_type,biz_id,key_value,envelope,status,attempts,
next_retry_at,sent_at,last_reason,last_error,created_at,updated_at
FROM im_kafka_outbox
WHERE status IN (?,?) AND (next_retry_at IS NULL OR next_retry_at<=?)
ORDER BY id ASC LIMIT ?`, t.KafkaOutboxPending, t.KafkaOutboxFailed, now, limit)
if err != nil {
return nil, err
}
out := make([]t.KafkaOutboxRecord, 0, len(rows))
for _, row := range rows {
rec := t.KafkaOutboxRecord{
ID: row.ID,
EventID: row.EventID,
Topic: row.Topic,
EventType: row.EventType,
BizType: row.BizType,
BizID: row.BizID,
Key: row.Key,
Envelope: row.Envelope,
Status: row.Status,
Attempts: row.Attempts,
LastReason: row.LastReason,
LastError: row.LastError,
CreatedAt: row.CreatedAt,
UpdatedAt: row.UpdatedAt,
}
if row.NextRetryAt.Valid {
rec.NextRetryAt = &row.NextRetryAt.Time
}
if row.SentAt.Valid {
rec.SentAt = &row.SentAt.Time
}
out = append(out, rec)
}
return out, nil
}
func (a *adapter) TenantKafkaOutboxMarkSent(id t.TenantID, eventID string, now time.Time) error {
tenantAdp, err := a.infrastructureAdapter(id)
if err != nil {
return err
}
if eventID == "" {
return t.ErrMalformed
}
if tenantAdp != a {
return tenantAdp.TenantKafkaOutboxMarkSent(id, eventID, now)
}
res, err := a.db.Exec(`UPDATE im_kafka_outbox SET status=?,sent_at=?,attempts=attempts+1,last_reason='',
last_error='',updated_at=? WHERE event_id=? AND status<>?`,
t.KafkaOutboxSent, now, now, eventID, t.KafkaOutboxSent)
if err != nil {
return err
}
if affected, _ := res.RowsAffected(); affected == 0 {
return t.ErrNotFound
}
return nil
}
func (a *adapter) TenantKafkaOutboxMarkFailed(id t.TenantID, eventID, reason, message string, nextRetryAt, now time.Time) error {
tenantAdp, err := a.infrastructureAdapter(id)
if err != nil {
return err
}
if eventID == "" {
return t.ErrMalformed
}
if tenantAdp != a {
return tenantAdp.TenantKafkaOutboxMarkFailed(id, eventID, reason, message, nextRetryAt, now)
}
res, err := a.db.Exec(`UPDATE im_kafka_outbox SET status=?,attempts=attempts+1,next_retry_at=?,
last_reason=?,last_error=?,updated_at=? WHERE event_id=? AND status<>?`,
t.KafkaOutboxFailed, nextRetryAt, truncateKafkaString(reason, 64), truncateKafkaString(message, 1000),
now, eventID, t.KafkaOutboxSent)
if err != nil {
return err
}
if affected, _ := res.RowsAffected(); affected == 0 {
return t.ErrNotFound
}
return nil
}
func scanKafkaDeadLetter(rows *sqlx.Rows) (*t.KafkaDeadLetter, error) {
var row struct {
ID int64 `db:"id"`
DeadID string `db:"dead_id"`
EventID string `db:"event_id"`
ParentEventID string `db:"parent_event_id"`
SourceTopic string `db:"source_topic"`
ConsumerGroup string `db:"consumer_group"`
EventType string `db:"event_type"`
BizType string `db:"biz_type"`
BizID string `db:"biz_id"`
Envelope []byte `db:"envelope"`
RetryCount int `db:"retry_count"`
LastReason string `db:"last_reason"`
LastError string `db:"last_error"`
Status string `db:"status"`
ReplayCount int `db:"replay_count"`
CreatedAt time.Time `db:"created_at"`
UpdatedAt time.Time `db:"updated_at"`
}
if err := rows.StructScan(&row); err != nil {
return nil, err
}
return &t.KafkaDeadLetter{
ID: row.ID,
DeadID: row.DeadID,
EventID: row.EventID,
ParentEventID: row.ParentEventID,
SourceTopic: row.SourceTopic,
ConsumerGroup: row.ConsumerGroup,
EventType: row.EventType,
BizType: row.BizType,
BizID: row.BizID,
Envelope: row.Envelope,
RetryCount: row.RetryCount,
LastReason: row.LastReason,
LastError: row.LastError,
Status: row.Status,
ReplayCount: row.ReplayCount,
CreatedAt: row.CreatedAt,
UpdatedAt: row.UpdatedAt,
}, nil
}
func (a *adapter) kafkaConsumeGet(eventID, consumerGroup string) (*t.KafkaConsumeRecord, error) {
row := struct {
EventID string `db:"event_id"`
ConsumerGroup string `db:"consumer_group"`
SourceTopic string `db:"source_topic"`
EventType string `db:"event_type"`
BizType string `db:"biz_type"`
BizID string `db:"biz_id"`
Status string `db:"status"`
RetryCount int `db:"retry_count"`
LastReason string `db:"last_reason"`
LastError string `db:"last_error"`
ProcessingDeadline *time.Time `db:"processing_deadline"`
SucceededAt *time.Time `db:"succeeded_at"`
DeadAt *time.Time `db:"dead_at"`
CreatedAt time.Time `db:"created_at"`
UpdatedAt time.Time `db:"updated_at"`
}{}
err := a.db.Get(&row, `SELECT event_id,consumer_group,source_topic,event_type,biz_type,biz_id,status,retry_count,
last_reason,last_error,processing_deadline,succeeded_at,dead_at,created_at,updated_at
FROM im_kafka_consume_log WHERE event_id=? AND consumer_group=?`, eventID, consumerGroup)
if err == sql.ErrNoRows {
return nil, t.ErrNotFound
}
if err != nil {
return nil, err
}
return &t.KafkaConsumeRecord{EventID: row.EventID, ConsumerGroup: row.ConsumerGroup, SourceTopic: row.SourceTopic,
EventType: row.EventType, BizType: row.BizType, BizID: row.BizID, Status: row.Status, RetryCount: row.RetryCount,
LastReason: row.LastReason, LastError: row.LastError, ProcessingDeadline: row.ProcessingDeadline,
SucceededAt: row.SucceededAt, DeadAt: row.DeadAt, CreatedAt: row.CreatedAt, UpdatedAt: row.UpdatedAt}, nil
}
func truncateKafkaString(value string, limit int) string {
if limit <= 0 {
return ""
}
runes := []rune(value)
if len(runes) <= limit {
return value
}
return string(runes[:limit])
}
func insertBusinessEventTx(tx *sqlx.Tx, eventID, targetType, target string, actor t.Uid, eventType string, payload any) error {
encoded, err := json.Marshal(payload)
if err != nil {
server/db/mysql/tenant_internal_admin_query.go
+126
View File
@@ -0,0 +1,126 @@
//go:build mysql
// +build mysql
package mysql
import (
"database/sql"
"encoding/json"
"strings"
"time"
"github.com/jmoiron/sqlx"
t "github.com/tinode/chat/server/store/types"
)
func (a *adapter) TenantInternalAdminQuery(id t.TenantID, query string, countQuery string, args []any, page, perPage int) (*t.InternalAdminQueryResult, error) {
tenantAdp, err := a.routedTenantAdapter(id)
if err != nil {
return nil, t.ErrMalformed
}
if tenantAdp != a {
return tenantAdp.TenantInternalAdminQuery(id, query, countQuery, args, page, perPage)
}
if strings.TrimSpace(query) == "" || strings.TrimSpace(countQuery) == "" {
return nil, t.ErrMalformed
}
if page <= 0 {
page = 1
}
if perPage <= 0 || perPage > 200 {
perPage = 20
}
offset := (page - 1) * perPage
ctx, cancel := a.getContext()
if cancel != nil {
defer cancel()
}
var total int64
if err = a.db.GetContext(ctx, &total, countQuery, args...); err != nil {
return nil, err
}
queryArgs := append(append([]any{}, args...), perPage, offset)
rows, err := a.db.QueryxContext(ctx, query+" LIMIT ? OFFSET ?", queryArgs...)
if err != nil {
return nil, err
}
defer rows.Close()
items, err := scanInternalAdminRows(rows)
if err != nil {
return nil, err
}
return &t.InternalAdminQueryResult{Items: items, Total: total, Page: page, PerPage: perPage}, nil
}
func (a *adapter) TenantInternalAdminQueryOne(id t.TenantID, query string, args []any) (map[string]any, error) {
tenantAdp, err := a.routedTenantAdapter(id)
if err != nil {
return nil, t.ErrMalformed
}
if tenantAdp != a {
return tenantAdp.TenantInternalAdminQueryOne(id, query, args)
}
if strings.TrimSpace(query) == "" {
return nil, t.ErrMalformed
}
ctx, cancel := a.getContext()
if cancel != nil {
defer cancel()
}
rows, err := a.db.QueryxContext(ctx, query+" LIMIT 1", args...)
if err != nil {
return nil, err
}
defer rows.Close()
items, err := scanInternalAdminRows(rows)
if err != nil {
return nil, err
}
if len(items) == 0 {
return nil, t.ErrNotFound
}
return items[0], nil
}
func scanInternalAdminRows(rows *sqlx.Rows) ([]map[string]any, error) {
items := []map[string]any{}
for rows.Next() {
row := map[string]any{}
if err := rows.MapScan(row); err != nil {
return nil, err
}
for key, value := range row {
row[key] = normalizeInternalAdminValue(value)
}
items = append(items, row)
}
return items, rows.Err()
}
func normalizeInternalAdminValue(value any) any {
switch v := value.(type) {
case nil:
return nil
case []byte:
raw := strings.TrimSpace(string(v))
if raw == "" {
return ""
}
if (strings.HasPrefix(raw, "{") && strings.HasSuffix(raw, "}")) || (strings.HasPrefix(raw, "[") && strings.HasSuffix(raw, "]")) {
var parsed any
if json.Unmarshal([]byte(raw), &parsed) == nil {
return parsed
}
}
return raw
case time.Time:
return v.UTC().Format(time.RFC3339Nano)
case sql.NullTime:
if v.Valid {
return v.Time.UTC().Format(time.RFC3339Nano)
}
return nil
default:
return value
}
}