feat: add kafka admin integration

server/admin_internal_config.go

@@ -0,0 +1,260 @@
+package main
+
+import (
+	"bytes"
+	"crypto/hmac"
+	"crypto/rand"
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/tinode/chat/server/logs"
+	"github.com/tinode/chat/server/store"
+	"github.com/tinode/chat/server/store/types"
+)
+
+type adminInternalAPIConfig struct {
+	Enabled     bool   `json:"enabled"`
+	BaseURL     string `json:"base_url"`
+	ServiceName string `json:"service_name"`
+	Secret      string `json:"secret"`
+	TimeoutMS   int    `json:"timeout_ms"`
+}
+
+type adminInternalClient struct {
+	config adminInternalAPIConfig
+	http   *http.Client
+}
+
+type adminInternalTenantDTO struct {
+	ID         types.TenantID    `json:"id"`
+	Code       string            `json:"code"`
+	Name       string            `json:"name"`
+	TenantDesc *string           `json:"tenant_desc,omitempty"`
+	DBName     string            `json:"db_name"`
+	DBDSN      string            `json:"db_dsn"`
+	State      types.TenantState `json:"state"`
+	CreatedAt  time.Time         `json:"created_at"`
+	CreatedBy  int64             `json:"created_by"`
+	UpdatedAt  time.Time         `json:"updated_at"`
+	UpdatedBy  int64             `json:"updated_by"`
+}
+
+func initAdminInternalAPI(raw json.RawMessage) error {
+	store.SetAdminInternalConfigProvider(nil)
+	if len(raw) == 0 || string(raw) == "null" {
+		return nil
+	}
+	var cfg adminInternalAPIConfig
+	if err := json.Unmarshal(raw, &cfg); err != nil {
+		return err
+	}
+	cfg.BaseURL = strings.TrimRight(strings.TrimSpace(cfg.BaseURL), "/")
+	cfg.ServiceName = strings.TrimSpace(cfg.ServiceName)
+	if cfg.ServiceName == "" {
+		cfg.ServiceName = "im-server"
+	}
+	if cfg.TimeoutMS <= 0 {
+		cfg.TimeoutMS = 3000
+	}
+	if !cfg.Enabled {
+		return nil
+	}
+	if cfg.BaseURL == "" || strings.TrimSpace(cfg.Secret) == "" {
+		return errors.New("admin_internal_api enabled but base_url or secret is empty")
+	}
+	store.SetAdminInternalConfigProvider(&adminInternalClient{
+		config: cfg,
+		http:   &http.Client{Timeout: time.Duration(cfg.TimeoutMS) * time.Millisecond},
+	})
+	logs.Info.Printf("Admin internal config client enabled: base_url=%s service=%s", cfg.BaseURL, cfg.ServiceName)
+	return nil
+}
+
+func (c *adminInternalClient) GetTenantByCode(code string) (*types.Tenant, error) {
+	var tenant adminInternalTenantDTO
+	if err := c.getJSON("/internal/im-config/tenants/by-code/"+url.PathEscape(strings.TrimSpace(code)), nil, &tenant); err != nil {
+		return nil, err
+	}
+	return tenant.toStoreTenant(), nil
+}
+
+func (c *adminInternalClient) GetTenantByID(id types.TenantID) (*types.Tenant, error) {
+	if !id.IsValid() {
+		return nil, types.ErrMalformed
+	}
+	var tenant adminInternalTenantDTO
+	if err := c.getJSON("/internal/im-config/tenants/by-id/"+strconv.FormatInt(int64(id), 10), nil, &tenant); err != nil {
+		return nil, err
+	}
+	return tenant.toStoreTenant(), nil
+}
+
+func (c *adminInternalClient) ListActiveTenants(limit int) ([]types.Tenant, error) {
+	query := url.Values{}
+	query.Set("state", "active")
+	if limit > 0 {
+		query.Set("limit", strconv.Itoa(limit))
+	}
+	var wrapper struct {
+		Items []adminInternalTenantDTO `json:"items"`
+	}
+	if err := c.getJSON("/internal/im-config/tenants", query, &wrapper); err != nil {
+		return nil, err
+	}
+	tenants := make([]types.Tenant, 0, len(wrapper.Items))
+	for _, item := range wrapper.Items {
+		if tenant := item.toStoreTenant(); tenant != nil {
+			tenants = append(tenants, *tenant)
+		}
+	}
+	return tenants, nil
+}
+
+func (c *adminInternalClient) RegistrationSMSEnabled() (bool, error) {
+	var wrapper struct {
+		Enabled bool `json:"enabled"`
+	}
+	if err := c.getJSON("/internal/im-config/registration/sms", nil, &wrapper); err != nil {
+		return true, err
+	}
+	return wrapper.Enabled, nil
+}
+
+func (c *adminInternalClient) ThirdPartyConfig(group string, keys ...string) (map[string]string, error) {
+	query := url.Values{}
+	normalized := make([]string, 0, len(keys))
+	for _, key := range keys {
+		if key = strings.TrimSpace(key); key != "" {
+			normalized = append(normalized, key)
+		}
+	}
+	if len(normalized) > 0 {
+		query.Set("keys", strings.Join(normalized, ","))
+	}
+	var wrapper struct {
+		Values map[string]string `json:"values"`
+	}
+	if err := c.getJSON("/internal/im-config/third-party/"+url.PathEscape(strings.TrimSpace(group)), query, &wrapper); err != nil {
+		return nil, err
+	}
+	if wrapper.Values == nil {
+		wrapper.Values = map[string]string{}
+	}
+	return wrapper.Values, nil
+}
+
+func (c *adminInternalClient) getJSON(path string, query url.Values, out any) error {
+	if c == nil || c.http == nil {
+		return types.ErrUnsupported
+	}
+	path = "/" + strings.TrimLeft(path, "/")
+	u, err := url.Parse(c.config.BaseURL + path)
+	if err != nil {
+		return err
+	}
+	if len(query) > 0 {
+		u.RawQuery = query.Encode()
+	}
+	req, err := http.NewRequest(http.MethodGet, u.String(), nil)
+	if err != nil {
+		return err
+	}
+	if err = c.signRequest(req, nil); err != nil {
+		return err
+	}
+	resp, err := c.http.Do(req)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+	body, err := io.ReadAll(io.LimitReader(resp.Body, 4<<20))
+	if err != nil {
+		return err
+	}
+	if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices {
+		return fmt.Errorf("admin internal api status=%d body=%s", resp.StatusCode, string(body))
+	}
+	var parsed struct {
+		Code int             `json:"code"`
+		Data json.RawMessage `json:"data"`
+	}
+	if err = json.Unmarshal(body, &parsed); err != nil {
+		return err
+	}
+	if parsed.Code != 0 {
+		return fmt.Errorf("admin internal api code=%d", parsed.Code)
+	}
+	if out == nil || len(parsed.Data) == 0 || string(parsed.Data) == "null" {
+		return nil
+	}
+	return json.Unmarshal(parsed.Data, out)
+}
+
+func (c *adminInternalClient) signRequest(req *http.Request, body []byte) error {
+	nonce, err := randomInternalNonce()
+	if err != nil {
+		return err
+	}
+	timestamp := strconv.FormatInt(time.Now().Unix(), 10)
+	bodyHash := sha256.Sum256(body)
+	signingText := strings.Join([]string{
+		req.Method,
+		req.URL.RequestURI(),
+		hex.EncodeToString(bodyHash[:]),
+		timestamp,
+		nonce,
+	}, "\n")
+	req.Header.Set(internalHeaderService, c.config.ServiceName)
+	req.Header.Set(internalHeaderTimestamp, timestamp)
+	req.Header.Set(internalHeaderNonce, nonce)
+	req.Header.Set(internalHeaderSignature, signInternalRequest(c.config.Secret, signingText))
+	req.Header.Set("Accept", "application/json")
+	if len(body) > 0 {
+		req.Body = io.NopCloser(bytes.NewReader(body))
+		req.ContentLength = int64(len(body))
+		req.Header.Set("Content-Type", "application/json")
+	}
+	return nil
+}
+
+func randomInternalNonce() (string, error) {
+	var raw [16]byte
+	if _, err := rand.Read(raw[:]); err != nil {
+		return "", err
+	}
+	return hex.EncodeToString(raw[:]), nil
+}
+
+func signInternalRequest(secret, signingText string) string {
+	mac := hmac.New(sha256.New, []byte(secret))
+	_, _ = mac.Write([]byte(signingText))
+	return hex.EncodeToString(mac.Sum(nil))
+}
+
+func (t adminInternalTenantDTO) toStoreTenant() *types.Tenant {
+	if t.ID.IsZero() && strings.TrimSpace(t.Code) == "" {
+		return nil
+	}
+	return &types.Tenant{
+		ID:         t.ID,
+		Code:       strings.TrimSpace(t.Code),
+		Name:       t.Name,
+		TenantDesc: t.TenantDesc,
+		DBName:     t.DBName,
+		DBDSN:      t.DBDSN,
+		State:      t.State,
+		CreatedAt:  t.CreatedAt,
+		CreatedBy:  t.CreatedBy,
+		UpdatedAt:  t.UpdatedAt,
+		UpdatedBy:  t.UpdatedBy,
+	}
+}

server/db/adapter.go

@@ -214,6 +214,7 @@ type Adapter interface {
 type TenantAdapter interface {
 	TenantGetByCode(code string) (*t.Tenant, error)
 	TenantGetByID(id t.TenantID) (*t.Tenant, error)
+	TenantListActive(limit int) ([]t.Tenant, error)
 }
 
 // RegistrationConfigAdapter reads global registration policy from the control database.
@@ -234,6 +235,11 @@ type TenantEntryAdapter interface {
 	TenantEntryFeedbackCreate(*t.TenantEntryFeedback) (*t.TenantEntryFeedback, error)
 }
 
+type TenantInternalAdminQueryAdapter interface {
+	TenantInternalAdminQuery(tenantID t.TenantID, query string, countQuery string, args []any, page, perPage int) (*t.InternalAdminQueryResult, error)
+	TenantInternalAdminQueryOne(tenantID t.TenantID, query string, args []any) (map[string]any, error)
+}
+
 // TenantRoutedAdapter can return a tenant-specific adapter, e.g. for the
 // "one tenant, one database" deployment model.
 type TenantRoutedAdapter interface {
@@ -249,6 +255,18 @@ type TenantHTTPInfrastructureAdapter interface {
 	TenantBusinessEventCreate(t.TenantID, *t.BusinessEvent) error
 	TenantBusinessEventListPending(t.TenantID, int, time.Time) ([]t.BusinessEvent, error)
 	TenantBusinessEventMarkDelivered(t.TenantID, string, time.Time) error
+	TenantKafkaConsumeBegin(t.TenantID, *t.KafkaConsumeRecord, time.Time) (*t.KafkaConsumeRecord, bool, error)
+	TenantKafkaConsumeSucceed(t.TenantID, string, string, time.Time) error
+	TenantKafkaConsumeFail(t.TenantID, string, string, string, string, time.Time) (*t.KafkaConsumeRecord, error)
+	TenantKafkaConsumeDead(t.TenantID, *t.KafkaDeadLetter, time.Time) error
+	TenantKafkaDeadLetterList(t.TenantID, t.KafkaDeadLetterListOptions) (*t.KafkaDeadLetterPage, error)
+	TenantKafkaDeadLetterGet(t.TenantID, string) (*t.KafkaDeadLetter, error)
+	TenantKafkaDeadLetterMark(t.TenantID, string, string, string, string, time.Time) error
+	TenantKafkaDeadLetterMarkReplay(t.TenantID, string, string, string, time.Time) (*t.KafkaDeadLetter, error)
+	TenantKafkaOutboxCreate(t.TenantID, *t.KafkaOutboxRecord) error
+	TenantKafkaOutboxListPending(t.TenantID, int, time.Time) ([]t.KafkaOutboxRecord, error)
+	TenantKafkaOutboxMarkSent(t.TenantID, string, time.Time) error
+	TenantKafkaOutboxMarkFailed(t.TenantID, string, string, string, time.Time, time.Time) error
 }
 
 // TenantRegistrationInviteAdapter stores invitation definitions in the

server/db/mysql/adapter.go

@@ -428,6 +428,31 @@ func (a *adapter) TenantGetByID(id t.TenantID) (*t.Tenant, error) {
 	return a.tenantByID(id)
 }
 
+func (a *adapter) TenantListActive(limit int) ([]t.Tenant, error) {
+	if limit <= 0 {
+		limit = 1000
+	}
+	ctx, cancel := a.getContext()
+	if cancel != nil {
+		defer cancel()
+	}
+	db := a.db
+	if a.registryDB != nil {
+		db = a.registryDB
+	}
+	var tenants []t.Tenant
+	err := db.SelectContext(ctx, &tenants,
+		`SELECT id, code, name, tenant_desc, IFNULL(db_name,'') AS db_name, IFNULL(db_dsn,'') AS db_dsn,
+		 state, created_at, created_by, updated_at, updated_by
+		 FROM im_tenant WHERE state=? ORDER BY id ASC LIMIT ?`, t.TenantStateActive, limit)
+	if isUnknownColumn(err) {
+		err = db.SelectContext(ctx, &tenants,
+			`SELECT id, code, name, tenant_desc, state, created_at, created_by, updated_at, updated_by
+			 FROM im_tenant WHERE state=? ORDER BY id ASC LIMIT ?`, t.TenantStateActive, limit)
+	}
+	return tenants, err
+}
+
 func (a *adapter) TenantAdapterFor(tenantID t.TenantID) (dbadapter.TenantBusinessAdapter, dbadapter.TenantAuthAdapter, error) {
 	if !a.tenantDBRouting {
 		return a, a, nil

server/db/mysql/infrastructure_schema.go

@@ -29,6 +29,52 @@ func ensureIMHTTPInfrastructureTables(exec sqlExecutor) error {
 			INDEX idx_im_business_events_outbox_pending(status, next_retry_at, id),
 			INDEX idx_im_business_events_outbox_target(target_type, target, id)
 		) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci`,
+		`CREATE TABLE IF NOT EXISTS im_kafka_consume_log (
+			id BIGINT NOT NULL AUTO_INCREMENT, event_id VARCHAR(96) NOT NULL,
+			consumer_group VARCHAR(128) NOT NULL, source_topic VARCHAR(128) NOT NULL,
+			event_type VARCHAR(96) NOT NULL DEFAULT '', biz_type VARCHAR(64) NOT NULL DEFAULT '',
+			biz_id VARCHAR(128) NOT NULL DEFAULT '', status VARCHAR(16) NOT NULL DEFAULT 'processing',
+			retry_count INT NOT NULL DEFAULT 0, last_reason VARCHAR(64) NOT NULL DEFAULT '',
+			last_error VARCHAR(1000) NOT NULL DEFAULT '', processing_deadline DATETIME(3),
+			succeeded_at DATETIME(3), dead_at DATETIME(3),
+			created_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3),
+			updated_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3) ON UPDATE CURRENT_TIMESTAMP(3),
+			PRIMARY KEY(id), UNIQUE INDEX uk_im_kafka_consume_log_event_group(event_id, consumer_group),
+			INDEX idx_im_kafka_consume_log_status(status, processing_deadline),
+			INDEX idx_im_kafka_consume_log_biz(biz_type, biz_id),
+			INDEX idx_im_kafka_consume_log_topic(source_topic, id)
+		) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci`,
+		`CREATE TABLE IF NOT EXISTS im_kafka_outbox (
+			id BIGINT NOT NULL AUTO_INCREMENT, event_id VARCHAR(96) NOT NULL,
+			topic VARCHAR(128) NOT NULL, event_type VARCHAR(96) NOT NULL,
+			biz_type VARCHAR(64) NOT NULL DEFAULT '', biz_id VARCHAR(128) NOT NULL DEFAULT '',
+			key_value VARCHAR(256) NOT NULL DEFAULT '', envelope JSON NOT NULL,
+			status VARCHAR(16) NOT NULL DEFAULT 'pending', attempts INT NOT NULL DEFAULT 0,
+			next_retry_at DATETIME(3), sent_at DATETIME(3), last_reason VARCHAR(64) NOT NULL DEFAULT '',
+			last_error VARCHAR(1000) NOT NULL DEFAULT '',
+			created_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3),
+			updated_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3) ON UPDATE CURRENT_TIMESTAMP(3),
+			PRIMARY KEY(id), UNIQUE INDEX uk_im_kafka_outbox_event(event_id),
+			INDEX idx_im_kafka_outbox_pending(status, next_retry_at, id),
+			INDEX idx_im_kafka_outbox_biz(biz_type, biz_id)
+		) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci`,
+		`CREATE TABLE IF NOT EXISTS im_kafka_dead_letters (
+			id BIGINT NOT NULL AUTO_INCREMENT, dead_id VARCHAR(96) NOT NULL,
+			event_id VARCHAR(96) NOT NULL, parent_event_id VARCHAR(96) NOT NULL DEFAULT '',
+			source_topic VARCHAR(128) NOT NULL, consumer_group VARCHAR(128) NOT NULL,
+			event_type VARCHAR(96) NOT NULL DEFAULT '', biz_type VARCHAR(64) NOT NULL DEFAULT '',
+			biz_id VARCHAR(128) NOT NULL DEFAULT '', envelope JSON NOT NULL,
+			retry_count INT NOT NULL DEFAULT 0, last_reason VARCHAR(64) NOT NULL DEFAULT '',
+			last_error VARCHAR(1000) NOT NULL DEFAULT '', status VARCHAR(24) NOT NULL DEFAULT 'dead',
+			replay_count INT NOT NULL DEFAULT 0, resolved_by VARCHAR(64) NOT NULL DEFAULT '',
+			resolved_at DATETIME(3), resolve_note VARCHAR(1000) NOT NULL DEFAULT '',
+			created_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3),
+			updated_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3) ON UPDATE CURRENT_TIMESTAMP(3),
+			PRIMARY KEY(id), UNIQUE INDEX uk_im_kafka_dead_letters_dead(dead_id),
+			INDEX idx_im_kafka_dead_letters_event(event_id),
+			INDEX idx_im_kafka_dead_letters_status(status, created_at),
+			INDEX idx_im_kafka_dead_letters_biz(biz_type, biz_id)
+		) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci`,
 		`CREATE TABLE IF NOT EXISTS im_realname_appeals (
 			id BIGINT NOT NULL, appeal_id VARCHAR(64) NOT NULL, user_id BIGINT NOT NULL,
 			reason VARCHAR(500) NOT NULL DEFAULT '', attachments_json JSON,

server/db/mysql/infrastructure_schema_test.go

@@ -12,13 +12,16 @@ func TestEnsureIMHTTPInfrastructureTablesUsesSelectedTenantDatabase(t *testing.T
 	if err := ensureIMHTTPInfrastructureTables(exec); err != nil {
 		t.Fatal(err)
 	}
-	if len(exec.statements) != 5 {
-		t.Fatalf("created %d infrastructure tables, want 5", len(exec.statements))
+	if len(exec.statements) != 8 {
+		t.Fatalf("created %d infrastructure tables, want 8", len(exec.statements))
 	}
 	joined := strings.ToLower(strings.Join(exec.statements, "\n"))
 	for _, table := range []string{
 		"im_http_idempotency",
 		"im_business_events_outbox",
+		"im_kafka_consume_log",
+		"im_kafka_outbox",
+		"im_kafka_dead_letters",
 		"im_realname_appeals",
 		"im_conversation_settings",
 		"im_topic_drafts",

server/db/mysql/tenant_adapter.go

@@ -2289,6 +2289,9 @@ func (a *adapter) TenantMessageSearch(id t.TenantID, topic string, uid t.Uid, qu
 	if limit <= 0 || limit > 100 {
 		limit = 50
 	}
+	if t.GetTopicCat(topic) == t.TopicCatGrp {
+		return a.tenantGroupMessageSearch(id, topic, uid, queryText, messageType, cursor, limit)
+	}
 	where := []string{"m.topic=?", "m.delid=0"}
 	args := []any{topic}
 	if cursor != "" {
@@ -2347,6 +2350,67 @@ func (a *adapter) TenantMessageSearch(id t.TenantID, topic string, uid t.Uid, qu
 	return msgs, nextCursor, nil
 }
 
+func (a *adapter) tenantGroupMessageSearch(id t.TenantID, topic string, uid t.Uid, queryText, messageType, cursor string, limit int) ([]t.Message, string, error) {
+	where := []string{"m.status!='deleted'", "m.topic=?"}
+	args := []any{topic}
+	if cursor != "" {
+		parsed, err := strconv.Atoi(cursor)
+		if err != nil || parsed <= 0 {
+			return nil, "", t.ErrMalformed
+		}
+		where = append(where, "m.seqid<?")
+		args = append(args, parsed)
+	}
+	queryText = strings.TrimSpace(queryText)
+	if queryText != "" {
+		where = append(where, "CAST(m.content AS CHAR) LIKE ?")
+		args = append(args, "%"+queryText+"%")
+	}
+	messageType = strings.TrimSpace(messageType)
+	if messageType != "" && messageType != "all" {
+		where = append(where, "m.msg_type=?")
+		args = append(args, messageType)
+	}
+	ctx, cancel := a.getContext()
+	if cancel != nil {
+		defer cancel()
+	}
+	queryArgs := append([]any{store.DecodeUid(uid)}, args...)
+	queryArgs = append(queryArgs, limit+1)
+	rows, err := a.db.QueryxContext(ctx,
+		"SELECT m.id,m.created_at AS createdat,m.updated_at AS updatedat,m.deleted_at AS deletedat,0 AS delid,m.seqid,m.topic,m.from_user AS `from`,IFNULL(m.client_msg_id,'') AS client_msg_id,m.version,m.recalled_at,m.recalled_by,m.head,m.content "+
+			"FROM im_group_messages AS m LEFT JOIN im_dellog AS d "+
+			"ON d.topic COLLATE utf8mb4_unicode_ci=m.topic AND m.seqid BETWEEN d.low AND d.hi-1 AND d.deletedfor=? "+
+			"WHERE "+strings.Join(where, " AND ")+" AND d.deletedfor IS NULL ORDER BY m.seqid DESC LIMIT ?",
+		queryArgs...)
+	if err != nil {
+		return nil, "", err
+	}
+	defer rows.Close()
+	msgs := make([]t.Message, 0, limit)
+	nextCursor := ""
+	for rows.Next() {
+		row := struct {
+			t.Message
+			RecalledBy sql.NullInt64 `db:"recalled_by"`
+		}{}
+		if err = rows.StructScan(&row); err != nil {
+			return nil, "", err
+		}
+		msg := normalizeGroupMessage(id, row.Message, row.RecalledBy)
+		msg.Head = kvMapFromJSON(msg.Head)
+		if len(msgs) == limit {
+			nextCursor = strconv.FormatInt(msg.SeqId, 10)
+			continue
+		}
+		msgs = append(msgs, msg)
+	}
+	if err = rows.Err(); err != nil {
+		return nil, "", err
+	}
+	return msgs, nextCursor, nil
+}
+
 func (a *adapter) TenantCallCreate(id t.TenantID, record *t.CallRecord) error {
 	if !a.tenantScopedDB {
 		tenantAdp, err := a.routedTenantAdapter(id)

server/db/mysql/tenant_infrastructure.go

@@ -4,8 +4,11 @@
 package mysql
 
 import (
+	"context"
 	"database/sql"
 	"encoding/json"
+	"strconv"
+	"strings"
 	"time"
 
 	"github.com/jmoiron/sqlx"
@@ -225,6 +228,512 @@ func (a *adapter) TenantBusinessEventMarkDelivered(id t.TenantID, eventID string
 	return nil
 }
 
+func (a *adapter) TenantKafkaConsumeBegin(id t.TenantID, record *t.KafkaConsumeRecord, deadline time.Time) (*t.KafkaConsumeRecord, bool, error) {
+	tenantAdp, err := a.infrastructureAdapter(id)
+	if err != nil {
+		return nil, false, err
+	}
+	if record == nil || record.EventID == "" || record.ConsumerGroup == "" || record.SourceTopic == "" {
+		return nil, false, t.ErrMalformed
+	}
+	if tenantAdp != a {
+		return tenantAdp.TenantKafkaConsumeBegin(id, record, deadline)
+	}
+	now := t.TimeNow()
+	_, err = a.db.Exec(`INSERT INTO im_kafka_consume_log
+		(event_id,consumer_group,source_topic,event_type,biz_type,biz_id,status,processing_deadline,created_at,updated_at)
+		VALUES(?,?,?,?,?,?,?,?,?,?)`, record.EventID, record.ConsumerGroup, record.SourceTopic, record.EventType,
+		record.BizType, record.BizID, t.KafkaConsumeProcessing, deadline, now, now)
+	if err == nil {
+		record.Status = t.KafkaConsumeProcessing
+		record.ProcessingDeadline = &deadline
+		record.CreatedAt = now
+		record.UpdatedAt = now
+		return record, true, nil
+	}
+	if !isDupe(err) {
+		return nil, false, err
+	}
+	existing, err := a.kafkaConsumeGet(record.EventID, record.ConsumerGroup)
+	if err != nil {
+		return nil, false, err
+	}
+	if existing.Status == t.KafkaConsumeSucceeded || existing.Status == t.KafkaConsumeDead {
+		return existing, false, nil
+	}
+	if existing.Status == t.KafkaConsumeProcessing && existing.ProcessingDeadline != nil && existing.ProcessingDeadline.After(now) {
+		return existing, false, nil
+	}
+	res, err := a.db.Exec(`UPDATE im_kafka_consume_log SET status=?,source_topic=?,event_type=?,biz_type=?,biz_id=?,
+		processing_deadline=?,updated_at=? WHERE event_id=? AND consumer_group=? AND status<>?`,
+		t.KafkaConsumeProcessing, record.SourceTopic, record.EventType, record.BizType, record.BizID,
+		deadline, now, record.EventID, record.ConsumerGroup, t.KafkaConsumeSucceeded)
+	if err != nil {
+		return nil, false, err
+	}
+	if affected, _ := res.RowsAffected(); affected == 0 {
+		current, getErr := a.kafkaConsumeGet(record.EventID, record.ConsumerGroup)
+		return current, false, getErr
+	}
+	current, err := a.kafkaConsumeGet(record.EventID, record.ConsumerGroup)
+	return current, err == nil, err
+}
+
+func (a *adapter) TenantKafkaConsumeSucceed(id t.TenantID, eventID, consumerGroup string, now time.Time) error {
+	tenantAdp, err := a.infrastructureAdapter(id)
+	if err != nil {
+		return err
+	}
+	if eventID == "" || consumerGroup == "" {
+		return t.ErrMalformed
+	}
+	if tenantAdp != a {
+		return tenantAdp.TenantKafkaConsumeSucceed(id, eventID, consumerGroup, now)
+	}
+	res, err := a.db.Exec(`UPDATE im_kafka_consume_log SET status=?,succeeded_at=?,processing_deadline=NULL,updated_at=?
+		WHERE event_id=? AND consumer_group=?`, t.KafkaConsumeSucceeded, now, now, eventID, consumerGroup)
+	if err != nil {
+		return err
+	}
+	if affected, _ := res.RowsAffected(); affected == 0 {
+		return t.ErrNotFound
+	}
+	return nil
+}
+
+func (a *adapter) TenantKafkaConsumeFail(id t.TenantID, eventID, consumerGroup, reason, message string, now time.Time) (*t.KafkaConsumeRecord, error) {
+	tenantAdp, err := a.infrastructureAdapter(id)
+	if err != nil {
+		return nil, err
+	}
+	if eventID == "" || consumerGroup == "" {
+		return nil, t.ErrMalformed
+	}
+	if tenantAdp != a {
+		return tenantAdp.TenantKafkaConsumeFail(id, eventID, consumerGroup, reason, message, now)
+	}
+	_, err = a.db.Exec(`UPDATE im_kafka_consume_log SET status=?,retry_count=retry_count+1,last_reason=?,
+		last_error=?,processing_deadline=NULL,updated_at=? WHERE event_id=? AND consumer_group=?`,
+		t.KafkaConsumeFailed, truncateKafkaString(reason, 64), truncateKafkaString(message, 1000), now, eventID, consumerGroup)
+	if err != nil {
+		return nil, err
+	}
+	return a.kafkaConsumeGet(eventID, consumerGroup)
+}
+
+func (a *adapter) TenantKafkaConsumeDead(id t.TenantID, dead *t.KafkaDeadLetter, now time.Time) error {
+	tenantAdp, err := a.infrastructureAdapter(id)
+	if err != nil {
+		return err
+	}
+	if dead == nil || dead.DeadID == "" || dead.EventID == "" || dead.ConsumerGroup == "" || dead.SourceTopic == "" || len(dead.Envelope) == 0 {
+		return t.ErrMalformed
+	}
+	if tenantAdp != a {
+		return tenantAdp.TenantKafkaConsumeDead(id, dead, now)
+	}
+	tx, err := a.db.BeginTxx(context.Background(), nil)
+	if err != nil {
+		return err
+	}
+	defer tx.Rollback()
+	if _, err = tx.Exec(`INSERT INTO im_kafka_dead_letters
+		(dead_id,event_id,parent_event_id,source_topic,consumer_group,event_type,biz_type,biz_id,envelope,
+		 retry_count,last_reason,last_error,status,replay_count,created_at,updated_at)
+		VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)
+		ON DUPLICATE KEY UPDATE last_reason=VALUES(last_reason),last_error=VALUES(last_error),updated_at=VALUES(updated_at)`,
+		dead.DeadID, dead.EventID, dead.ParentEventID, dead.SourceTopic, dead.ConsumerGroup, dead.EventType,
+		dead.BizType, dead.BizID, dead.Envelope, dead.RetryCount, truncateKafkaString(dead.LastReason, 64),
+		truncateKafkaString(dead.LastError, 1000), t.KafkaDeadStatusDead, dead.ReplayCount, now, now); err != nil {
+		return err
+	}
+	if _, err = tx.Exec(`UPDATE im_kafka_consume_log SET status=?,dead_at=?,processing_deadline=NULL,last_reason=?,
+		last_error=?,updated_at=? WHERE event_id=? AND consumer_group=?`,
+		t.KafkaConsumeDead, now, truncateKafkaString(dead.LastReason, 64), truncateKafkaString(dead.LastError, 1000),
+		now, dead.EventID, dead.ConsumerGroup); err != nil {
+		return err
+	}
+	return tx.Commit()
+}
+
+func (a *adapter) TenantKafkaDeadLetterList(id t.TenantID, opts t.KafkaDeadLetterListOptions) (*t.KafkaDeadLetterPage, error) {
+	tenantAdp, err := a.infrastructureAdapter(id)
+	if err != nil {
+		return nil, err
+	}
+	if tenantAdp != a {
+		return tenantAdp.TenantKafkaDeadLetterList(id, opts)
+	}
+	limit := opts.Limit
+	if limit <= 0 || limit > 200 {
+		limit = 50
+	}
+	where := []string{"1=1"}
+	args := []any{}
+	if opts.Status != "" {
+		where = append(where, "status=?")
+		args = append(args, opts.Status)
+	}
+	if opts.BizType != "" {
+		where = append(where, "biz_type=?")
+		args = append(args, opts.BizType)
+	}
+	if opts.BizID != "" {
+		where = append(where, "biz_id=?")
+		args = append(args, opts.BizID)
+	}
+	if opts.EventType != "" {
+		where = append(where, "event_type=?")
+		args = append(args, opts.EventType)
+	}
+	if opts.Cursor != "" {
+		where = append(where, "id<?")
+		args = append(args, opts.Cursor)
+	}
+	args = append(args, limit+1)
+	rows, err := a.db.Queryx(`SELECT id,dead_id,event_id,parent_event_id,source_topic,consumer_group,event_type,biz_type,biz_id,
+		envelope,retry_count,last_reason,last_error,status,replay_count,created_at,updated_at
+		FROM im_kafka_dead_letters WHERE `+strings.Join(where, " AND ")+` ORDER BY id DESC LIMIT ?`, args...)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	page := &t.KafkaDeadLetterPage{Items: make([]t.KafkaDeadLetter, 0, limit)}
+	for rows.Next() {
+		dead, err := scanKafkaDeadLetter(rows)
+		if err != nil {
+			return nil, err
+		}
+		if len(page.Items) == limit {
+			page.NextCursor = strconv.FormatInt(dead.ID, 10)
+			continue
+		}
+		page.Items = append(page.Items, *dead)
+	}
+	return page, rows.Err()
+}
+
+func (a *adapter) TenantKafkaDeadLetterGet(id t.TenantID, deadID string) (*t.KafkaDeadLetter, error) {
+	tenantAdp, err := a.infrastructureAdapter(id)
+	if err != nil {
+		return nil, err
+	}
+	if strings.TrimSpace(deadID) == "" {
+		return nil, t.ErrMalformed
+	}
+	if tenantAdp != a {
+		return tenantAdp.TenantKafkaDeadLetterGet(id, deadID)
+	}
+	rows, err := a.db.Queryx(`SELECT id,dead_id,event_id,parent_event_id,source_topic,consumer_group,event_type,biz_type,biz_id,
+		envelope,retry_count,last_reason,last_error,status,replay_count,created_at,updated_at
+		FROM im_kafka_dead_letters WHERE dead_id=?`, strings.TrimSpace(deadID))
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	if rows.Next() {
+		return scanKafkaDeadLetter(rows)
+	}
+	if err = rows.Err(); err != nil {
+		return nil, err
+	}
+	return nil, t.ErrNotFound
+}
+
+func (a *adapter) TenantKafkaDeadLetterMark(id t.TenantID, deadID, status, resolvedBy, note string, now time.Time) error {
+	tenantAdp, err := a.infrastructureAdapter(id)
+	if err != nil {
+		return err
+	}
+	deadID = strings.TrimSpace(deadID)
+	status = strings.TrimSpace(status)
+	if deadID == "" || (status != t.KafkaDeadStatusIgnored && status != t.KafkaDeadStatusResolved && status != t.KafkaDeadStatusReplayed) {
+		return t.ErrMalformed
+	}
+	if tenantAdp != a {
+		return tenantAdp.TenantKafkaDeadLetterMark(id, deadID, status, resolvedBy, note, now)
+	}
+	res, err := a.db.Exec(`UPDATE im_kafka_dead_letters SET status=?,resolved_by=?,resolved_at=?,resolve_note=?,updated_at=?
+		WHERE dead_id=? AND status IN (?,?,?)`, status, truncateKafkaString(resolvedBy, 64), now,
+		truncateKafkaString(note, 1000), now, deadID, t.KafkaDeadStatusDead, t.KafkaDeadStatusPendingReplay, t.KafkaDeadStatusReplayed)
+	if err != nil {
+		return err
+	}
+	if affected, _ := res.RowsAffected(); affected == 0 {
+		return t.ErrNotFound
+	}
+	return nil
+}
+
+func (a *adapter) TenantKafkaDeadLetterMarkReplay(id t.TenantID, deadID, resolvedBy, note string, now time.Time) (*t.KafkaDeadLetter, error) {
+	tenantAdp, err := a.infrastructureAdapter(id)
+	if err != nil {
+		return nil, err
+	}
+	deadID = strings.TrimSpace(deadID)
+	if deadID == "" {
+		return nil, t.ErrMalformed
+	}
+	if tenantAdp != a {
+		return tenantAdp.TenantKafkaDeadLetterMarkReplay(id, deadID, resolvedBy, note, now)
+	}
+	tx, err := a.db.BeginTxx(context.Background(), nil)
+	if err != nil {
+		return nil, err
+	}
+	defer tx.Rollback()
+	var row struct {
+		EventID       string `db:"event_id"`
+		ConsumerGroup string `db:"consumer_group"`
+	}
+	if err = tx.Get(&row, `SELECT event_id,consumer_group FROM im_kafka_dead_letters WHERE dead_id=? FOR UPDATE`, deadID); err != nil {
+		if err == sql.ErrNoRows {
+			return nil, t.ErrNotFound
+		}
+		return nil, err
+	}
+	res, err := tx.Exec(`UPDATE im_kafka_dead_letters SET status=?,replay_count=replay_count+1,resolved_by=?,
+		resolved_at=?,resolve_note=?,updated_at=? WHERE dead_id=? AND status IN (?,?)`,
+		t.KafkaDeadStatusPendingReplay, truncateKafkaString(resolvedBy, 64), now, truncateKafkaString(note, 1000),
+		now, deadID, t.KafkaDeadStatusDead, t.KafkaDeadStatusPendingReplay)
+	if err != nil {
+		return nil, err
+	}
+	if affected, _ := res.RowsAffected(); affected == 0 {
+		return nil, t.ErrNotFound
+	}
+	if _, err = tx.Exec(`DELETE FROM im_kafka_consume_log WHERE event_id=? AND consumer_group=?`,
+		row.EventID, row.ConsumerGroup); err != nil {
+		return nil, err
+	}
+	if err = tx.Commit(); err != nil {
+		return nil, err
+	}
+	return a.TenantKafkaDeadLetterGet(id, deadID)
+}
+
+func (a *adapter) TenantKafkaOutboxCreate(id t.TenantID, record *t.KafkaOutboxRecord) error {
+	tenantAdp, err := a.infrastructureAdapter(id)
+	if err != nil {
+		return err
+	}
+	if record == nil || record.EventID == "" || record.Topic == "" || record.EventType == "" || len(record.Envelope) == 0 {
+		return t.ErrMalformed
+	}
+	if tenantAdp != a {
+		return tenantAdp.TenantKafkaOutboxCreate(id, record)
+	}
+	now := t.TimeNow()
+	_, err = a.db.Exec(`INSERT INTO im_kafka_outbox
+		(event_id,topic,event_type,biz_type,biz_id,key_value,envelope,status,next_retry_at,created_at,updated_at)
+		VALUES(?,?,?,?,?,?,?,?,?,?,?)
+		ON DUPLICATE KEY UPDATE updated_at=updated_at`,
+		record.EventID, record.Topic, record.EventType, record.BizType, record.BizID, record.Key,
+		record.Envelope, t.KafkaOutboxPending, record.NextRetryAt, now, now)
+	return err
+}
+
+func (a *adapter) TenantKafkaOutboxListPending(id t.TenantID, limit int, now time.Time) ([]t.KafkaOutboxRecord, error) {
+	tenantAdp, err := a.infrastructureAdapter(id)
+	if err != nil {
+		return nil, err
+	}
+	if limit <= 0 {
+		limit = 100
+	}
+	if tenantAdp != a {
+		return tenantAdp.TenantKafkaOutboxListPending(id, limit, now)
+	}
+	rows := []struct {
+		ID          int64        `db:"id"`
+		EventID     string       `db:"event_id"`
+		Topic       string       `db:"topic"`
+		EventType   string       `db:"event_type"`
+		BizType     string       `db:"biz_type"`
+		BizID       string       `db:"biz_id"`
+		Key         string       `db:"key_value"`
+		Envelope    []byte       `db:"envelope"`
+		Status      string       `db:"status"`
+		Attempts    int          `db:"attempts"`
+		NextRetryAt sql.NullTime `db:"next_retry_at"`
+		SentAt      sql.NullTime `db:"sent_at"`
+		LastReason  string       `db:"last_reason"`
+		LastError   string       `db:"last_error"`
+		CreatedAt   time.Time    `db:"created_at"`
+		UpdatedAt   time.Time    `db:"updated_at"`
+	}{}
+	err = a.db.Select(&rows, `SELECT id,event_id,topic,event_type,biz_type,biz_id,key_value,envelope,status,attempts,
+		next_retry_at,sent_at,last_reason,last_error,created_at,updated_at
+		FROM im_kafka_outbox
+		WHERE status IN (?,?) AND (next_retry_at IS NULL OR next_retry_at<=?)
+		ORDER BY id ASC LIMIT ?`, t.KafkaOutboxPending, t.KafkaOutboxFailed, now, limit)
+	if err != nil {
+		return nil, err
+	}
+	out := make([]t.KafkaOutboxRecord, 0, len(rows))
+	for _, row := range rows {
+		rec := t.KafkaOutboxRecord{
+			ID:         row.ID,
+			EventID:    row.EventID,
+			Topic:      row.Topic,
+			EventType:  row.EventType,
+			BizType:    row.BizType,
+			BizID:      row.BizID,
+			Key:        row.Key,
+			Envelope:   row.Envelope,
+			Status:     row.Status,
+			Attempts:   row.Attempts,
+			LastReason: row.LastReason,
+			LastError:  row.LastError,
+			CreatedAt:  row.CreatedAt,
+			UpdatedAt:  row.UpdatedAt,
+		}
+		if row.NextRetryAt.Valid {
+			rec.NextRetryAt = &row.NextRetryAt.Time
+		}
+		if row.SentAt.Valid {
+			rec.SentAt = &row.SentAt.Time
+		}
+		out = append(out, rec)
+	}
+	return out, nil
+}
+
+func (a *adapter) TenantKafkaOutboxMarkSent(id t.TenantID, eventID string, now time.Time) error {
+	tenantAdp, err := a.infrastructureAdapter(id)
+	if err != nil {
+		return err
+	}
+	if eventID == "" {
+		return t.ErrMalformed
+	}
+	if tenantAdp != a {
+		return tenantAdp.TenantKafkaOutboxMarkSent(id, eventID, now)
+	}
+	res, err := a.db.Exec(`UPDATE im_kafka_outbox SET status=?,sent_at=?,attempts=attempts+1,last_reason='',
+		last_error='',updated_at=? WHERE event_id=? AND status<>?`,
+		t.KafkaOutboxSent, now, now, eventID, t.KafkaOutboxSent)
+	if err != nil {
+		return err
+	}
+	if affected, _ := res.RowsAffected(); affected == 0 {
+		return t.ErrNotFound
+	}
+	return nil
+}
+
+func (a *adapter) TenantKafkaOutboxMarkFailed(id t.TenantID, eventID, reason, message string, nextRetryAt, now time.Time) error {
+	tenantAdp, err := a.infrastructureAdapter(id)
+	if err != nil {
+		return err
+	}
+	if eventID == "" {
+		return t.ErrMalformed
+	}
+	if tenantAdp != a {
+		return tenantAdp.TenantKafkaOutboxMarkFailed(id, eventID, reason, message, nextRetryAt, now)
+	}
+	res, err := a.db.Exec(`UPDATE im_kafka_outbox SET status=?,attempts=attempts+1,next_retry_at=?,
+		last_reason=?,last_error=?,updated_at=? WHERE event_id=? AND status<>?`,
+		t.KafkaOutboxFailed, nextRetryAt, truncateKafkaString(reason, 64), truncateKafkaString(message, 1000),
+		now, eventID, t.KafkaOutboxSent)
+	if err != nil {
+		return err
+	}
+	if affected, _ := res.RowsAffected(); affected == 0 {
+		return t.ErrNotFound
+	}
+	return nil
+}
+
+func scanKafkaDeadLetter(rows *sqlx.Rows) (*t.KafkaDeadLetter, error) {
+	var row struct {
+		ID            int64     `db:"id"`
+		DeadID        string    `db:"dead_id"`
+		EventID       string    `db:"event_id"`
+		ParentEventID string    `db:"parent_event_id"`
+		SourceTopic   string    `db:"source_topic"`
+		ConsumerGroup string    `db:"consumer_group"`
+		EventType     string    `db:"event_type"`
+		BizType       string    `db:"biz_type"`
+		BizID         string    `db:"biz_id"`
+		Envelope      []byte    `db:"envelope"`
+		RetryCount    int       `db:"retry_count"`
+		LastReason    string    `db:"last_reason"`
+		LastError     string    `db:"last_error"`
+		Status        string    `db:"status"`
+		ReplayCount   int       `db:"replay_count"`
+		CreatedAt     time.Time `db:"created_at"`
+		UpdatedAt     time.Time `db:"updated_at"`
+	}
+	if err := rows.StructScan(&row); err != nil {
+		return nil, err
+	}
+	return &t.KafkaDeadLetter{
+		ID:            row.ID,
+		DeadID:        row.DeadID,
+		EventID:       row.EventID,
+		ParentEventID: row.ParentEventID,
+		SourceTopic:   row.SourceTopic,
+		ConsumerGroup: row.ConsumerGroup,
+		EventType:     row.EventType,
+		BizType:       row.BizType,
+		BizID:         row.BizID,
+		Envelope:      row.Envelope,
+		RetryCount:    row.RetryCount,
+		LastReason:    row.LastReason,
+		LastError:     row.LastError,
+		Status:        row.Status,
+		ReplayCount:   row.ReplayCount,
+		CreatedAt:     row.CreatedAt,
+		UpdatedAt:     row.UpdatedAt,
+	}, nil
+}
+
+func (a *adapter) kafkaConsumeGet(eventID, consumerGroup string) (*t.KafkaConsumeRecord, error) {
+	row := struct {
+		EventID            string     `db:"event_id"`
+		ConsumerGroup      string     `db:"consumer_group"`
+		SourceTopic        string     `db:"source_topic"`
+		EventType          string     `db:"event_type"`
+		BizType            string     `db:"biz_type"`
+		BizID              string     `db:"biz_id"`
+		Status             string     `db:"status"`
+		RetryCount         int        `db:"retry_count"`
+		LastReason         string     `db:"last_reason"`
+		LastError          string     `db:"last_error"`
+		ProcessingDeadline *time.Time `db:"processing_deadline"`
+		SucceededAt        *time.Time `db:"succeeded_at"`
+		DeadAt             *time.Time `db:"dead_at"`
+		CreatedAt          time.Time  `db:"created_at"`
+		UpdatedAt          time.Time  `db:"updated_at"`
+	}{}
+	err := a.db.Get(&row, `SELECT event_id,consumer_group,source_topic,event_type,biz_type,biz_id,status,retry_count,
+		last_reason,last_error,processing_deadline,succeeded_at,dead_at,created_at,updated_at
+		FROM im_kafka_consume_log WHERE event_id=? AND consumer_group=?`, eventID, consumerGroup)
+	if err == sql.ErrNoRows {
+		return nil, t.ErrNotFound
+	}
+	if err != nil {
+		return nil, err
+	}
+	return &t.KafkaConsumeRecord{EventID: row.EventID, ConsumerGroup: row.ConsumerGroup, SourceTopic: row.SourceTopic,
+		EventType: row.EventType, BizType: row.BizType, BizID: row.BizID, Status: row.Status, RetryCount: row.RetryCount,
+		LastReason: row.LastReason, LastError: row.LastError, ProcessingDeadline: row.ProcessingDeadline,
+		SucceededAt: row.SucceededAt, DeadAt: row.DeadAt, CreatedAt: row.CreatedAt, UpdatedAt: row.UpdatedAt}, nil
+}
+
+func truncateKafkaString(value string, limit int) string {
+	if limit <= 0 {
+		return ""
+	}
+	runes := []rune(value)
+	if len(runes) <= limit {
+		return value
+	}
+	return string(runes[:limit])
+}
+
 func insertBusinessEventTx(tx *sqlx.Tx, eventID, targetType, target string, actor t.Uid, eventType string, payload any) error {
 	encoded, err := json.Marshal(payload)
 	if err != nil {

server/db/mysql/tenant_internal_admin_query.go

@@ -0,0 +1,126 @@
+//go:build mysql
+// +build mysql
+
+package mysql
+
+import (
+	"database/sql"
+	"encoding/json"
+	"strings"
+	"time"
+
+	"github.com/jmoiron/sqlx"
+	t "github.com/tinode/chat/server/store/types"
+)
+
+func (a *adapter) TenantInternalAdminQuery(id t.TenantID, query string, countQuery string, args []any, page, perPage int) (*t.InternalAdminQueryResult, error) {
+	tenantAdp, err := a.routedTenantAdapter(id)
+	if err != nil {
+		return nil, t.ErrMalformed
+	}
+	if tenantAdp != a {
+		return tenantAdp.TenantInternalAdminQuery(id, query, countQuery, args, page, perPage)
+	}
+	if strings.TrimSpace(query) == "" || strings.TrimSpace(countQuery) == "" {
+		return nil, t.ErrMalformed
+	}
+	if page <= 0 {
+		page = 1
+	}
+	if perPage <= 0 || perPage > 200 {
+		perPage = 20
+	}
+	offset := (page - 1) * perPage
+	ctx, cancel := a.getContext()
+	if cancel != nil {
+		defer cancel()
+	}
+	var total int64
+	if err = a.db.GetContext(ctx, &total, countQuery, args...); err != nil {
+		return nil, err
+	}
+	queryArgs := append(append([]any{}, args...), perPage, offset)
+	rows, err := a.db.QueryxContext(ctx, query+" LIMIT ? OFFSET ?", queryArgs...)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	items, err := scanInternalAdminRows(rows)
+	if err != nil {
+		return nil, err
+	}
+	return &t.InternalAdminQueryResult{Items: items, Total: total, Page: page, PerPage: perPage}, nil
+}
+
+func (a *adapter) TenantInternalAdminQueryOne(id t.TenantID, query string, args []any) (map[string]any, error) {
+	tenantAdp, err := a.routedTenantAdapter(id)
+	if err != nil {
+		return nil, t.ErrMalformed
+	}
+	if tenantAdp != a {
+		return tenantAdp.TenantInternalAdminQueryOne(id, query, args)
+	}
+	if strings.TrimSpace(query) == "" {
+		return nil, t.ErrMalformed
+	}
+	ctx, cancel := a.getContext()
+	if cancel != nil {
+		defer cancel()
+	}
+	rows, err := a.db.QueryxContext(ctx, query+" LIMIT 1", args...)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	items, err := scanInternalAdminRows(rows)
+	if err != nil {
+		return nil, err
+	}
+	if len(items) == 0 {
+		return nil, t.ErrNotFound
+	}
+	return items[0], nil
+}
+
+func scanInternalAdminRows(rows *sqlx.Rows) ([]map[string]any, error) {
+	items := []map[string]any{}
+	for rows.Next() {
+		row := map[string]any{}
+		if err := rows.MapScan(row); err != nil {
+			return nil, err
+		}
+		for key, value := range row {
+			row[key] = normalizeInternalAdminValue(value)
+		}
+		items = append(items, row)
+	}
+	return items, rows.Err()
+}
+
+func normalizeInternalAdminValue(value any) any {
+	switch v := value.(type) {
+	case nil:
+		return nil
+	case []byte:
+		raw := strings.TrimSpace(string(v))
+		if raw == "" {
+			return ""
+		}
+		if (strings.HasPrefix(raw, "{") && strings.HasSuffix(raw, "}")) || (strings.HasPrefix(raw, "[") && strings.HasSuffix(raw, "]")) {
+			var parsed any
+			if json.Unmarshal([]byte(raw), &parsed) == nil {
+				return parsed
+			}
+		}
+		return raw
+	case time.Time:
+		return v.UTC().Format(time.RFC3339Nano)
+	case sql.NullTime:
+		if v.Valid {
+			return v.Time.UTC().Format(time.RFC3339Nano)
+		}
+		return nil
+	default:
+		return value
+	}
+}

server/hdl_files.go

@@ -10,6 +10,7 @@
 package main
 
 import (
+	"context"
 	"encoding/base64"
 	"encoding/json"
 	"errors"
@@ -349,6 +350,9 @@ func largeFileReceiveHTTP(wrt http.ResponseWriter, req *http.Request) {
 		writeHttpResponse(decodeStoreError(err, msgID, now, nil), err)
 		return
 	}
+	if err = enqueueKafkaFileUploaded(req.Context(), principal.TenantID, fdef); err != nil {
+		logs.Warn.Printf("media upload: failed to enqueue file lifecycle event file=%s: %v", fdef.Id, err)
+	}
 
 	params := map[string]string{"url": url}
 	if globals.mediaGcPeriod > 0 {
@@ -628,6 +632,15 @@ func (*grpcNodeServer) LargeFileReceive(stream pbx.Node_LargeFileReceiveServer)
 			Size:     size,
 		},
 	})
+	if err == nil {
+		if fdef, finishErr := store.Files.FinishUpload(principal.TenantID, fdef, true, size); finishErr == nil {
+			if lifecycleErr := enqueueKafkaFileUploaded(context.Background(), principal.TenantID, fdef); lifecycleErr != nil {
+				logs.Warn.Printf("media upload: failed to enqueue grpc file lifecycle event file=%s: %v", fdef.Id, lifecycleErr)
+			}
+		} else {
+			err = finishErr
+		}
+	}
 	logs.Info.Println("media upload: ok", fdef.Id, fdef.Location, err)
 	return err
 }

server/im_http.go

@@ -68,7 +68,7 @@ const (
 	imQRCodeMaxExpiresInSeconds      = 7 * 86400
 	imPublicUploadMaxBytes           = 10 << 20
 	groupMaxAdminCount               = 10
-	monthlyReportPreviewMaxRunes     = 120
+	reportPreviewMaxRunes            = 120
 )
 
 type imHTTPResponse struct {
@@ -170,6 +170,10 @@ func serveIMHTTPIdempotent(wrt http.ResponseWriter, req *http.Request, next http
 		next.ServeHTTP(wrt, req)
 		return
 	}
+	if req.Method == http.MethodPost && strings.TrimSpace(req.URL.Path) == "/v1/im/tenants/resolve" {
+		next.ServeHTTP(wrt, req)
+		return
+	}
 	if !strings.HasPrefix(strings.TrimSpace(req.Header.Get("Authorization")), "Bearer ") {
 		next.ServeHTTP(wrt, req)
 		return
@@ -396,6 +400,16 @@ type imFavoritesForwardRequest struct {
 	Comment      string   `json:"comment,omitempty"`
 }
 
+type imMessagesForwardRequest struct {
+	Mode    string `json:"mode"`
+	Sources []struct {
+		Topic string `json:"topic"`
+		Seq   int64  `json:"seq,string"`
+	} `json:"sources"`
+	TargetTopics []string `json:"target_topics"`
+	Comment      string   `json:"comment,omitempty"`
+}
+
 type imFriendRequestCreateRequest struct {
 	TargetUserID string `json:"target_user_id"`
 	Message      string `json:"message,omitempty"`
@@ -1886,6 +1900,8 @@ func serveIMMeetings(wrt http.ResponseWriter, req *http.Request) {
 			return
 		}
 	}
+	publishIMWorkbenchAndAudit(req.Context(), principal.TenantID, principal.Uid, "meeting", "meeting.created", meeting.MeetingID,
+		meetingParticipantUIDs(meeting), "会议通知", meeting.Title, map[string]any{"meeting_id": meeting.MeetingID, "version": meeting.Version, "state": meeting.State})
 	writeIMSuccess(wrt, req, meetingPayload(meeting, principal.Uid))
 }
 
@@ -1979,6 +1995,8 @@ func serveIMMeetingUpdate(wrt http.ResponseWriter, req *http.Request, meetingID
 			return
 		}
 	}
+	publishIMWorkbenchAndAudit(req.Context(), principal.TenantID, principal.Uid, "meeting", "meeting.updated", meeting.MeetingID,
+		meetingParticipantUIDs(meeting), "会议通知", meeting.Title, map[string]any{"meeting_id": meeting.MeetingID, "version": meeting.Version, "state": meeting.State})
 	writeIMSuccess(wrt, req, meetingPayload(meeting, principal.Uid))
 }
 
@@ -2006,6 +2024,8 @@ func serveIMMeetingCancel(wrt http.ResponseWriter, req *http.Request, meetingID
 			return
 		}
 	}
+	publishIMWorkbenchAndAudit(req.Context(), principal.TenantID, principal.Uid, "meeting", "meeting.canceled", meeting.MeetingID,
+		meetingParticipantUIDs(meeting), "会议通知", meeting.Title, map[string]any{"meeting_id": meeting.MeetingID, "version": meeting.Version, "state": meeting.State})
 	writeIMSuccess(wrt, req, meetingPayload(meeting, principal.Uid))
 }
 
@@ -2039,6 +2059,8 @@ func serveIMMeetingReminders(wrt http.ResponseWriter, req *http.Request, meeting
 			return
 		}
 	}
+	publishIMWorkbenchAndAudit(req.Context(), principal.TenantID, principal.Uid, "meeting", "meeting.reminded", meetingID,
+		meetingParticipantUserIDs(participants), "会议提醒", meetingID, map[string]any{"meeting_id": meetingID, "reminded_count": len(participants)})
 	writeIMSuccess(wrt, req, map[string]any{
 		"meeting_id": meetingID,
 		"reminded":   meetingParticipantPayloads(participants),
@@ -2072,6 +2094,8 @@ func serveIMMeetingMyResponse(wrt http.ResponseWriter, req *http.Request, meetin
 			return
 		}
 	}
+	publishIMWorkbenchAndAudit(req.Context(), principal.TenantID, principal.Uid, "meeting", "meeting.response.updated", meeting.MeetingID,
+		meetingParticipantUIDs(meeting), "会议通知", meeting.Title, map[string]any{"meeting_id": meeting.MeetingID, "version": meeting.Version, "state": meeting.State})
 	writeIMSuccess(wrt, req, meetingPayload(meeting, principal.Uid))
 }
 
@@ -2126,6 +2150,8 @@ func serveIMDailyReports(wrt http.ResponseWriter, req *http.Request) {
 			writeIMRelationError(wrt, req, err)
 			return
 		}
+		publishIMWorkbenchAndAudit(req.Context(), principal.TenantID, principal.Uid, "daily_report", "daily_report.draft.created", report.ReportID,
+			[]types.Uid{principal.Uid}, "日报", report.ReportDate.Format("2006-01-02"), map[string]any{"report_id": report.ReportID, "version": report.Version, "status": report.Status})
 		writeIMSuccess(wrt, req, dailyReportDetailPayload(report, principal.Uid))
 	default:
 		writeIMError(wrt, req, http.StatusMethodNotAllowed, 30002, imReasonMethodNotAllowed, "method not allowed", nil)
@@ -2172,6 +2198,8 @@ func serveIMDailyReportGet(wrt http.ResponseWriter, req *http.Request, reportID
 		writeIMRelationError(wrt, req, err)
 		return
 	}
+	publishIMWorkbenchAndAudit(req.Context(), principal.TenantID, principal.Uid, "daily_report", "daily_report.draft.saved", report.ReportID,
+		[]types.Uid{principal.Uid}, "日报", report.ReportDate.Format("2006-01-02"), map[string]any{"report_id": report.ReportID, "version": report.Version, "status": report.Status})
 	writeIMSuccess(wrt, req, dailyReportDetailPayload(report, principal.Uid))
 }
 
@@ -2197,6 +2225,8 @@ func serveIMDailyReportSaveDraft(wrt http.ResponseWriter, req *http.Request, rep
 		writeIMRelationError(wrt, req, err)
 		return
 	}
+	publishIMWorkbenchAndAudit(req.Context(), principal.TenantID, principal.Uid, "daily_report", "daily_report.submitted", report.ReportID,
+		[]types.Uid{principal.Uid}, "日报已提交", report.ReportDate.Format("2006-01-02"), map[string]any{"report_id": report.ReportID, "version": report.Version, "status": report.Status})
 	writeIMSuccess(wrt, req, dailyReportDetailPayload(report, principal.Uid))
 }
 
@@ -2288,6 +2318,8 @@ func serveIMDailyReportReminders(wrt http.ResponseWriter, req *http.Request) {
 	for _, uid := range reminded {
 		items = append(items, map[string]any{"user_id": uid.UserId(), "status": "queued"})
 	}
+	publishIMWorkbenchAndAudit(req.Context(), principal.TenantID, principal.Uid, "daily_report", "daily_report.reminded", date.Format("2006-01-02"),
+		reminded, "日报提醒", date.Format("2006-01-02"), map[string]any{"date": date.Format("2006-01-02"), "reminded_count": len(reminded)})
 	writeIMSuccess(wrt, req, map[string]any{"date": date.Format("2006-01-02"), "items": items})
 }
 
@@ -2332,6 +2364,8 @@ func serveIMMonthlyReports(wrt http.ResponseWriter, req *http.Request) {
 			writeIMRelationError(wrt, req, err)
 			return
 		}
+		publishIMWorkbenchAndAudit(req.Context(), principal.TenantID, principal.Uid, "monthly_report", "monthly_report.draft.created", report.ReportID,
+			[]types.Uid{principal.Uid}, "月报", report.Month, map[string]any{"report_id": report.ReportID, "version": report.Version, "status": report.Status})
 		writeIMSuccess(wrt, req, monthlyReportDetailPayload(report, principal.Uid))
 	default:
 		writeIMError(wrt, req, http.StatusMethodNotAllowed, 30002, imReasonMethodNotAllowed, "method not allowed", nil)
@@ -2377,6 +2411,8 @@ func serveIMMonthlyReportGet(wrt http.ResponseWriter, req *http.Request, reportI
 		writeIMRelationError(wrt, req, err)
 		return
 	}
+	publishIMWorkbenchAndAudit(req.Context(), principal.TenantID, principal.Uid, "monthly_report", "monthly_report.draft.saved", report.ReportID,
+		[]types.Uid{principal.Uid}, "月报", report.Month, map[string]any{"report_id": report.ReportID, "version": report.Version, "status": report.Status})
 	writeIMSuccess(wrt, req, monthlyReportDetailPayload(report, principal.Uid))
 }
 
@@ -2403,6 +2439,8 @@ func serveIMMonthlyReportSaveDraft(wrt http.ResponseWriter, req *http.Request, r
 		writeIMRelationError(wrt, req, err)
 		return
 	}
+	publishIMWorkbenchAndAudit(req.Context(), principal.TenantID, principal.Uid, "monthly_report", "monthly_report.submitted", report.ReportID,
+		[]types.Uid{principal.Uid}, "月报已提交", report.Month, map[string]any{"report_id": report.ReportID, "version": report.Version, "status": report.Status})
 	writeIMSuccess(wrt, req, monthlyReportDetailPayload(report, principal.Uid))
 }
 
@@ -2426,6 +2464,8 @@ func serveIMMonthlyReportSubmit(wrt http.ResponseWriter, req *http.Request, repo
 		writeIMRelationError(wrt, req, err)
 		return
 	}
+	publishIMWorkbenchAndAudit(req.Context(), principal.TenantID, principal.Uid, "monthly_report", "monthly_report.reopened", report.ReportID,
+		[]types.Uid{principal.Uid}, "月报已重开", report.Month, map[string]any{"report_id": report.ReportID, "version": report.Version, "status": report.Status})
 	writeIMSuccess(wrt, req, monthlyReportDetailPayload(report, principal.Uid))
 }
 
@@ -2541,6 +2581,8 @@ func serveIMMailboxSubmissions(wrt http.ResponseWriter, req *http.Request) {
 		writeIMRelationError(wrt, req, err)
 		return
 	}
+	publishIMWorkbenchAndAudit(req.Context(), principal.TenantID, principal.Uid, "mailbox", "mailbox.submitted", submission.SubmissionID,
+		[]types.Uid{principal.Uid}, "总经理信箱", submission.Subject, map[string]any{"submission_id": submission.SubmissionID, "status": submission.Status, "anonymous": submission.Anonymous})
 	writeIMSuccess(wrt, req, mailboxSubmissionResultPayload(submission))
 }
 
@@ -3201,7 +3243,7 @@ func serveIMFavoriteForward(wrt http.ResponseWriter, req *http.Request) {
 				},
 				Content: favoriteForwardContent(item, comment),
 			}
-			saved, err := store.Messages.Save(principal.TenantID, msg, nil, true)
+			saved, err := saveMessageMaybeKafka(req.Context(), principal.TenantID, msg, nil, true)
 			if err != nil {
 				results = append(results, map[string]any{"target_topic": target, "favorite_id": favoriteID, "status": "failed", "reason": relationErrorReason(err)})
 				continue
@@ -3212,6 +3254,107 @@ func serveIMFavoriteForward(wrt http.ResponseWriter, req *http.Request) {
 	writeIMSuccess(wrt, req, map[string]any{"items": results})
 }
 
+func serveIMMessageForward(wrt http.ResponseWriter, req *http.Request) {
+	if req.Method != http.MethodPost {
+		writeIMError(wrt, req, http.StatusMethodNotAllowed, 30002, imReasonMethodNotAllowed, "method not allowed", nil)
+		return
+	}
+	principal, ok := authenticateIMHTTPBearer(wrt, req)
+	if !ok {
+		return
+	}
+	var body imMessagesForwardRequest
+	if !decodeIMJSONBody(wrt, req, &body) {
+		return
+	}
+	mode := strings.TrimSpace(body.Mode)
+	if mode == "" {
+		mode = "separate"
+	}
+	if mode != "separate" {
+		writeIMError(wrt, req, http.StatusNotImplemented, 30004, "NOT_IMPLEMENTED", "forward bundle is not implemented", map[string]any{"field": "mode"})
+		return
+	}
+	if len(body.Sources) == 0 || len(body.Sources) > 100 || len(body.TargetTopics) == 0 || len(body.TargetTopics) > 20 {
+		writeIMError(wrt, req, http.StatusBadRequest, 30000, imReasonMalformedRequest, "malformed request", nil)
+		return
+	}
+	sourceReadable := make(map[string]error)
+	targetWritable := make(map[string]error)
+	items := make([]map[string]any, 0, len(body.TargetTopics)*len(body.Sources))
+	failed := make([]map[string]any, 0)
+	for _, rawTarget := range body.TargetTopics {
+		target := strings.TrimSpace(rawTarget)
+		if target == "" {
+			for _, source := range body.Sources {
+				failed = append(failed, forwardFailurePayload(source.Topic, source.Seq, target, imReasonMalformedRequest))
+			}
+			continue
+		}
+		targetErr, ok := targetWritable[target]
+		if !ok {
+			targetErr = requireTopicWriter(principal, target)
+			targetWritable[target] = targetErr
+		}
+		if targetErr != nil {
+			for _, source := range body.Sources {
+				failed = append(failed, forwardFailurePayload(source.Topic, source.Seq, target, relationErrorReason(targetErr)))
+			}
+			continue
+		}
+		for _, source := range body.Sources {
+			sourceTopic := strings.TrimSpace(source.Topic)
+			if sourceTopic == "" || source.Seq <= 0 {
+				failed = append(failed, forwardFailurePayload(sourceTopic, source.Seq, target, imReasonMalformedRequest))
+				continue
+			}
+			readErr, ok := sourceReadable[sourceTopic]
+			if !ok {
+				readErr = requireTopicReader(principal, sourceTopic)
+				sourceReadable[sourceTopic] = readErr
+			}
+			if readErr != nil {
+				failed = append(failed, forwardFailurePayload(sourceTopic, source.Seq, target, relationErrorReason(readErr)))
+				continue
+			}
+			msg, err := store.Messages.GetOne(principal.TenantID, sourceTopic, source.Seq)
+			if err != nil {
+				failed = append(failed, forwardFailurePayload(sourceTopic, source.Seq, target, relationErrorReason(err)))
+				continue
+			}
+			if msg == nil {
+				failed = append(failed, forwardFailurePayload(sourceTopic, source.Seq, target, "NOT_FOUND"))
+				continue
+			}
+			if messageAvailability(*msg) != "available" {
+				failed = append(failed, forwardFailurePayload(sourceTopic, source.Seq, target, "MESSAGE_UNAVAILABLE"))
+				continue
+			}
+			clientMid := "fwd_" + store.Store.GetUidString()
+			forwarded := &types.Message{
+				TenantID: principal.TenantID,
+				Topic:    target,
+				From:     principal.Uid.String(),
+				Head:     forwardedMessageHead(msg, clientMid),
+				Content:  forwardedMessageContent(msg, body.Comment),
+			}
+			saved, err := saveMessageMaybeKafka(req.Context(), principal.TenantID, forwarded, nil, true)
+			if err != nil {
+				failed = append(failed, forwardFailurePayload(sourceTopic, source.Seq, target, relationErrorReason(err)))
+				continue
+			}
+			items = append(items, map[string]any{
+				"source":       map[string]any{"topic": sourceTopic, "seq": source.Seq},
+				"target_topic": target,
+				"client_mid":   clientMid,
+				"seq":          saved.SeqId,
+				"status":       "sent",
+			})
+		}
+	}
+	writeIMSuccess(wrt, req, map[string]any{"items": items, "failed": failed})
+}
+
 func serveIMTopicMessageAction(wrt http.ResponseWriter, req *http.Request) {
 	switch {
 	case strings.HasSuffix(req.URL.Path, "/draft"):
@@ -4500,6 +4643,9 @@ func serveIMGroupFiles(wrt http.ResponseWriter, req *http.Request, topic, tail s
 			writeIMRelationError(wrt, req, err)
 			return
 		}
+		if err = enqueueKafkaGroupFileLifecycle(req.Context(), principal.TenantID, kafkaFileLifecycleGroupFileCreate, file, principal.Uid); err != nil {
+			logs.Warn.Printf("im http: failed to enqueue group file lifecycle event topic=%s file=%s: %v", topic, file.FileID, err)
+		}
 		writeIMSuccess(wrt, req, groupFilePayload(file))
 	case strings.HasPrefix(tail, "files/") && req.Method == http.MethodDelete:
 		groupFileID := strings.TrimSpace(strings.TrimPrefix(tail, "files/"))
@@ -4513,6 +4659,9 @@ func serveIMGroupFiles(wrt http.ResponseWriter, req *http.Request, topic, tail s
 			writeIMRelationError(wrt, req, err)
 			return
 		}
+		if err = enqueueKafkaGroupFileLifecycle(req.Context(), principal.TenantID, kafkaFileLifecycleGroupFileDelete, file, principal.Uid); err != nil {
+			logs.Warn.Printf("im http: failed to enqueue group file lifecycle event topic=%s file=%s: %v", topic, file.FileID, err)
+		}
 		writeIMSuccess(wrt, req, groupFilePayload(file))
 	default:
 		writeIMError(wrt, req, http.StatusMethodNotAllowed, 30002, imReasonMethodNotAllowed, "method not allowed", nil)
@@ -7134,6 +7283,33 @@ func meetingParticipantPayloads(participants []types.MeetingParticipant) []map[s
 	return items
 }
 
+func meetingParticipantUIDs(meeting *types.Meeting) []types.Uid {
+	if meeting == nil {
+		return nil
+	}
+	return meetingParticipantUserIDs(meeting.Participants)
+}
+
+func meetingParticipantUserIDs(participants []types.MeetingParticipant) []types.Uid {
+	items := make([]types.Uid, 0, len(participants))
+	for _, participant := range participants {
+		if !participant.UserID.IsZero() {
+			items = append(items, participant.UserID)
+		}
+	}
+	return items
+}
+
+func publishIMWorkbenchAndAudit(ctx context.Context, tenantID types.TenantID, actor types.Uid, module, action, resourceID string,
+	targets []types.Uid, title, summary string, data map[string]any) {
+	if err := enqueueKafkaWorkbenchEvent(ctx, tenantID, actor, module, action, resourceID, targets, title, summary, data); err != nil {
+		logs.Warn.Printf("im http: failed to enqueue workbench event action=%s resource=%s: %v", action, resourceID, err)
+	}
+	if err := enqueueKafkaAuditEvent(ctx, tenantID, "user", actor, action, module, resourceID, "succeeded", "", data); err != nil {
+		logs.Warn.Printf("im http: failed to enqueue audit event action=%s resource=%s: %v", action, resourceID, err)
+	}
+}
+
 func meetingPermissionsPayload(meeting *types.Meeting, viewer types.Uid) map[string]any {
 	isOrganizer := meeting.OrganizerID == viewer
 	isScheduled := meeting.State == types.MeetingStateScheduled
@@ -7168,7 +7344,7 @@ func dailyReportListPayload(report types.DailyReport) map[string]any {
 		"report_id":        report.ReportID,
 		"author_id":        report.AuthorID.UserId(),
 		"report_date":      report.ReportDate.Format("2006-01-02"),
-		"content_short":    monthlyReportPreview(report.Content),
+		"content_short":    reportPreview(report.Content),
 		"status":           report.Status,
 		"draft_updated_at": report.UpdatedAt.UTC().Format(time.RFC3339Nano),
 		"read_state":       report.ReadState,
@@ -7211,8 +7387,8 @@ func monthlyReportListPayload(report types.MonthlyReport) map[string]any {
 		"report_id":          report.ReportID,
 		"author_id":          report.AuthorID.UserId(),
 		"month":              report.Month,
-		"work_summary_short": monthlyReportPreview(report.WorkSummary),
-		"next_plan_short":    monthlyReportPreview(report.NextPlan),
+		"work_summary_short": reportPreview(report.WorkSummary),
+		"next_plan_short":    reportPreview(report.NextPlan),
 		"status":             report.Status,
 		"read_state":         report.ReadState,
 		"version":            report.Version,
@@ -7225,13 +7401,13 @@ func monthlyReportListPayload(report types.MonthlyReport) map[string]any {
 	return data
 }
 
-func monthlyReportPreview(text string) string {
+func reportPreview(text string) string {
 	text = strings.Join(strings.Fields(text), " ")
 	runes := []rune(text)
-	if len(runes) <= monthlyReportPreviewMaxRunes {
+	if len(runes) <= reportPreviewMaxRunes {
 		return text
 	}
-	return string(runes[:monthlyReportPreviewMaxRunes]) + "..."
+	return string(runes[:reportPreviewMaxRunes]) + "..."
 }
 
 func monthlyReportDetailPayload(report *types.MonthlyReport, viewer types.Uid) map[string]any {
@@ -7826,6 +8002,20 @@ func requireTopicWriter(principal *imPrincipal, topic string) error {
 	return nil
 }
 
+func requireTopicReader(principal *imPrincipal, topic string) error {
+	if principal == nil || topic == "" {
+		return types.ErrMalformed
+	}
+	sub, err := store.Subs.Get(principal.TenantID, topic, principal.Uid, false)
+	if err != nil {
+		return err
+	}
+	if sub == nil || !(sub.ModeGiven & sub.ModeWant).IsReader() {
+		return types.ErrPermissionDenied
+	}
+	return nil
+}
+
 func copyStringAnyMap(in map[string]any) map[string]any {
 	out := make(map[string]any, len(in))
 	for key, value := range in {
@@ -7956,6 +8146,49 @@ func favoriteForwardContent(item *types.FavoriteItem, comment string) map[string
 	return payload
 }
 
+func forwardFailurePayload(topic string, seq int64, target, reason string) map[string]any {
+	return map[string]any{
+		"source":       map[string]any{"topic": strings.TrimSpace(topic), "seq": seq},
+		"target_topic": strings.TrimSpace(target),
+		"status":       "failed",
+		"reason":       reason,
+	}
+}
+
+func forwardedMessageHead(msg *types.Message, clientMid string) types.KVMap {
+	head := types.KVMap{}
+	if msg != nil {
+		for key, value := range msg.Head {
+			head[key] = value
+		}
+		msgType := imMessageType(msg)
+		if strings.TrimSpace(fmt.Sprint(head["type"])) == "" || fmt.Sprint(head["type"]) == "<nil>" {
+			head["type"] = msgType
+		}
+		if strings.TrimSpace(fmt.Sprint(head["x-im-type"])) == "" || fmt.Sprint(head["x-im-type"]) == "<nil>" {
+			head["x-im-type"] = msgType
+		}
+		head["x-im-forward-source"] = map[string]any{"topic": msg.Topic, "seq": msg.SeqId, "from": msg.From}
+	}
+	head["x-im-forwarded"] = true
+	head["x-im-client-mid"] = clientMid
+	return head
+}
+
+func forwardedMessageContent(msg *types.Message, comment string) any {
+	if strings.TrimSpace(comment) == "" {
+		if msg == nil {
+			return nil
+		}
+		return msg.Content
+	}
+	return map[string]any{
+		"forwarded": true,
+		"comment":   strings.TrimSpace(comment),
+		"snapshot":  favoriteSnapshot(msg),
+	}
+}
+
 func classifyIMURLRisk(rawURL string) (decision, riskLevel, reason string) {
 	if rawURL == "" || len(rawURL) > 2048 {
 		return "block", "high", "invalid_url"
@@ -8802,14 +9035,10 @@ func peerFromP2PTopic(topic string, uid types.Uid) types.Uid {
 	if types.GetTopicCat(topic) != types.TopicCatP2P || uid.IsZero() {
 		return types.ZeroUid
 	}
-	prefix := "p2p"
-	body := strings.TrimPrefix(topic, prefix)
-	uidLen := len(uid.String())
-	if len(body) < 2*uidLen {
+	left, right, err := types.ParseP2P(topic)
+	if err != nil {
 		return types.ZeroUid
 	}
-	left := types.ParseUid(body[:uidLen])
-	right := types.ParseUid(body[uidLen:])
 	if left == uid {
 		return right
 	}

server/im_http_test.go

@@ -40,6 +40,7 @@ type testVerificationStore struct {
 type testHTTPInfrastructureStore struct {
 	records map[string]*types.HTTPIdempotencyRecord
 	events  []types.BusinessEvent
+	outbox  []types.KafkaOutboxRecord
 }
 
 type testTenantEntryStore struct {
@@ -147,6 +148,57 @@ func (s *testHTTPInfrastructureStore) MarkEventDelivered(_ types.TenantID, _ str
 	return nil
 }
 
+func (s *testHTTPInfrastructureStore) BeginKafkaConsume(_ types.TenantID, record *types.KafkaConsumeRecord, _ time.Time) (*types.KafkaConsumeRecord, bool, error) {
+	copy := *record
+	copy.Status = types.KafkaConsumeProcessing
+	return &copy, true, nil
+}
+
+func (s *testHTTPInfrastructureStore) MarkKafkaConsumeSucceeded(_ types.TenantID, _, _ string, _ time.Time) error {
+	return nil
+}
+
+func (s *testHTTPInfrastructureStore) MarkKafkaConsumeFailed(_ types.TenantID, eventID, group, reason, message string, _ time.Time) (*types.KafkaConsumeRecord, error) {
+	return &types.KafkaConsumeRecord{EventID: eventID, ConsumerGroup: group, Status: types.KafkaConsumeFailed, RetryCount: 1, LastReason: reason, LastError: message}, nil
+}
+
+func (s *testHTTPInfrastructureStore) MarkKafkaConsumeDead(_ types.TenantID, _ *types.KafkaDeadLetter, _ time.Time) error {
+	return nil
+}
+
+func (s *testHTTPInfrastructureStore) ListKafkaDeadLetters(_ types.TenantID, _ types.KafkaDeadLetterListOptions) (*types.KafkaDeadLetterPage, error) {
+	return &types.KafkaDeadLetterPage{}, nil
+}
+
+func (s *testHTTPInfrastructureStore) GetKafkaDeadLetter(_ types.TenantID, _ string) (*types.KafkaDeadLetter, error) {
+	return nil, types.ErrNotFound
+}
+
+func (s *testHTTPInfrastructureStore) MarkKafkaDeadLetter(_ types.TenantID, _, _, _, _ string, _ time.Time) error {
+	return nil
+}
+
+func (s *testHTTPInfrastructureStore) MarkKafkaDeadLetterReplay(_ types.TenantID, deadID, _, _ string, _ time.Time) (*types.KafkaDeadLetter, error) {
+	return &types.KafkaDeadLetter{DeadID: deadID, Status: types.KafkaDeadStatusPendingReplay}, nil
+}
+
+func (s *testHTTPInfrastructureStore) CreateKafkaOutbox(_ types.TenantID, record *types.KafkaOutboxRecord) error {
+	s.outbox = append(s.outbox, *record)
+	return nil
+}
+
+func (s *testHTTPInfrastructureStore) ListPendingKafkaOutbox(_ types.TenantID, _ int, _ time.Time) ([]types.KafkaOutboxRecord, error) {
+	return append([]types.KafkaOutboxRecord(nil), s.outbox...), nil
+}
+
+func (s *testHTTPInfrastructureStore) MarkKafkaOutboxSent(_ types.TenantID, _ string, _ time.Time) error {
+	return nil
+}
+
+func (s *testHTTPInfrastructureStore) MarkKafkaOutboxFailed(_ types.TenantID, _, _, _ string, _, _ time.Time) error {
+	return nil
+}
+
 type testRelationsStore struct {
 	context        *types.RelationshipContext
 	created        *types.FriendRequest
@@ -1261,6 +1313,23 @@ func TestServeIMTenantResolveSuccessFromHeader(t *testing.T) {
 	}
 }
 
+func TestIMHTTPProtocolDoesNotRequireIdempotencyForTenantResolve(t *testing.T) {
+	next := http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+		writeIMSuccess(w, req, map[string]any{"ok": true})
+	})
+	req := httptest.NewRequest(http.MethodPost, "/v1/im/tenants/resolve", nil)
+	req.Header.Set("X-Tinode-APIKey", makeTestAPIKey(t))
+	req.Header.Set("X-IM-Schema", "1")
+	req.Header.Set("Authorization", "Bearer test-token")
+	rec := httptest.NewRecorder()
+
+	withIMHTTPProtocol("/v1/im/", next).ServeHTTP(rec, req)
+
+	if rec.Code != http.StatusOK {
+		t.Fatalf("status: got %d, want %d; body=%s", rec.Code, http.StatusOK, rec.Body.String())
+	}
+}
+
 func TestServeIMTenantResolveUsesForwardedPublicHost(t *testing.T) {
 	oldTenants := store.Tenants
 	oldServingAt := globals.servingAt
@@ -2348,6 +2417,42 @@ func TestServeIMDailyReportsCreateAndList(t *testing.T) {
 	}
 }
 
+func TestServeIMDailyReportsListIncludesContentPreview(t *testing.T) {
+	ctrl := gomock.NewController(t)
+	defer ctrl.Finish()
+	expectIMBearer(t, ctrl, "token-1", 7, 42)
+	oldDailyReports := store.DailyReports
+	now := time.Now().UTC().Round(time.Millisecond)
+	reports := &testDailyReportsStore{report: &types.DailyReport{
+		ReportID:   "dr_test",
+		AuthorID:   42,
+		ReportDate: time.Date(2026, 8, 14, 0, 0, 0, 0, time.UTC),
+		Content:    strings.Repeat("日报", 80),
+		Status:     types.DailyReportStatusDraft,
+		ReadState:  "unread",
+		Version:    1,
+		CreatedAt:  now,
+		UpdatedAt:  now,
+	}}
+	store.DailyReports = reports
+	defer func() { store.DailyReports = oldDailyReports }()
+
+	req := httptest.NewRequest(http.MethodGet, "/v1/im/reports/daily?scope=mine", nil)
+	req.Header.Set("Authorization", "Bearer token-1")
+	rec := httptest.NewRecorder()
+	serveIMDailyReports(rec, req)
+
+	if rec.Code != http.StatusOK {
+		t.Fatalf("status: got %d, want %d; body=%s", rec.Code, http.StatusOK, rec.Body.String())
+	}
+	data := decodeIMHTTPResponse(t, rec).Data.(map[string]any)
+	items := data["items"].([]any)
+	item := items[0].(map[string]any)
+	if item["content_short"] != strings.Repeat("日报", 60)+"..." {
+		t.Fatalf("unexpected daily report preview: %#v", item["content_short"])
+	}
+}
+
 func TestServeIMDailyReportDraftAndSubmit(t *testing.T) {
 	ctrl := gomock.NewController(t)
 	defer ctrl.Finish()
@@ -2419,42 +2524,6 @@ func TestServeIMDailyReportStatisticsAndReminders(t *testing.T) {
 	}
 }
 
-func TestServeIMDailyReportsListIncludesContentPreview(t *testing.T) {
-	ctrl := gomock.NewController(t)
-	defer ctrl.Finish()
-	expectIMBearer(t, ctrl, "token-1", 7, 42)
-	oldDailyReports := store.DailyReports
-	now := time.Now().UTC().Round(time.Millisecond)
-	reports := &testDailyReportsStore{report: &types.DailyReport{
-		ReportID:   "dr_test",
-		AuthorID:   42,
-		ReportDate: time.Date(2026, 8, 14, 0, 0, 0, 0, time.UTC),
-		Content:    strings.Repeat("日报", 80),
-		Status:     types.DailyReportStatusDraft,
-		ReadState:  "unread",
-		Version:    1,
-		CreatedAt:  now,
-		UpdatedAt:  now,
-	}}
-	store.DailyReports = reports
-	defer func() { store.DailyReports = oldDailyReports }()
-
-	req := httptest.NewRequest(http.MethodGet, "/v1/im/reports/daily?scope=mine", nil)
-	req.Header.Set("Authorization", "Bearer token-1")
-	rec := httptest.NewRecorder()
-	serveIMDailyReports(rec, req)
-
-	if rec.Code != http.StatusOK {
-		t.Fatalf("status: got %d, want %d; body=%s", rec.Code, http.StatusOK, rec.Body.String())
-	}
-	data := decodeIMHTTPResponse(t, rec).Data.(map[string]any)
-	items := data["items"].([]any)
-	item := items[0].(map[string]any)
-	if item["content_short"] != strings.Repeat("日报", 60)+"..." {
-		t.Fatalf("unexpected daily report preview: %#v", item["content_short"])
-	}
-}
-
 func TestServeIMMonthlyReportsCreate(t *testing.T) {
 	ctrl := gomock.NewController(t)
 	defer ctrl.Finish()
@@ -3988,6 +4057,83 @@ func TestServeIMFavoriteForwardRejectsUnauthorizedTarget(t *testing.T) {
 	}
 }
 
+func TestServeIMMessageForwardSeparateSavesMessages(t *testing.T) {
+	ctrl := gomock.NewController(t)
+	defer ctrl.Finish()
+	mockStore := expectIMBearer(t, ctrl, "token-1", 7, 42)
+	mockStore.EXPECT().GetUidString().Return("mid-1")
+
+	oldSubs := store.Subs
+	oldMessages := store.Messages
+	subs := mock_store.NewMockSubsPersistenceInterface(ctrl)
+	messages := mock_store.NewMockMessagesPersistenceInterface(ctrl)
+	store.Subs = subs
+	store.Messages = messages
+	t.Cleanup(func() {
+		store.Subs = oldSubs
+		store.Messages = oldMessages
+	})
+
+	subs.EXPECT().Get(types.TenantID(7), "grpTarget", types.Uid(42), false).Return(&types.Subscription{
+		TenantID:  7,
+		User:      types.Uid(42).String(),
+		Topic:     "grpTarget",
+		ModeWant:  types.ModeCPublic,
+		ModeGiven: types.ModeCPublic,
+	}, nil)
+	subs.EXPECT().Get(types.TenantID(7), "grpSource", types.Uid(42), false).Return(&types.Subscription{
+		TenantID:  7,
+		User:      types.Uid(42).String(),
+		Topic:     "grpSource",
+		ModeWant:  types.ModeCPublic,
+		ModeGiven: types.ModeCPublic,
+	}, nil)
+	messages.EXPECT().GetOne(types.TenantID(7), "grpSource", int64(3)).Return(&types.Message{
+		TenantID: 7,
+		Topic:    "grpSource",
+		SeqId:    3,
+		From:     types.Uid(99).String(),
+		Head:     types.KVMap{"x-im-type": "text"},
+		Content:  map[string]any{"text": "hello"},
+		ObjHeader: types.ObjHeader{
+			CreatedAt: time.Date(2026, 8, 13, 9, 0, 0, 0, time.UTC),
+		},
+	}, nil)
+	messages.EXPECT().Save(types.TenantID(7), gomock.Any(), gomock.Nil(), true).DoAndReturn(
+		func(_ types.TenantID, msg *types.Message, _ []string, _ bool) (*types.MessageSaveResult, error) {
+			if msg.Topic != "grpTarget" || msg.From != types.Uid(42).String() {
+				t.Fatalf("unexpected forwarded message: %#v", msg)
+			}
+			if msg.Head["x-im-forwarded"] != true || msg.Head["x-im-client-mid"] != "fwd_mid-1" {
+				t.Fatalf("forward metadata not set: %#v", msg.Head)
+			}
+			if msg.Content.(map[string]any)["text"] != "hello" {
+				t.Fatalf("unexpected content: %#v", msg.Content)
+			}
+			return &types.MessageSaveResult{SeqId: 21, MarkedReadBySender: true}, nil
+		})
+
+	req := httptest.NewRequest(http.MethodPost, "/v1/im/messages/forward", bytes.NewBufferString(`{"mode":"separate","sources":[{"topic":"grpSource","seq":"3"}],"target_topics":["grpTarget"]}`))
+	req.Header.Set("Authorization", "Bearer token-1")
+	rec := httptest.NewRecorder()
+	serveIMMessageForward(rec, req)
+	if rec.Code != http.StatusOK {
+		t.Fatalf("forward status: got %d body=%s", rec.Code, rec.Body.String())
+	}
+	data := decodeIMHTTPResponse(t, rec).Data.(map[string]any)
+	items := data["items"].([]any)
+	if len(items) != 1 {
+		t.Fatalf("unexpected forward response: %#v", data)
+	}
+	got := items[0].(map[string]any)
+	if got["status"] != "sent" || got["seq"].(float64) != 21 || got["client_mid"] != "fwd_mid-1" {
+		t.Fatalf("unexpected forward item: %#v", got)
+	}
+	if failed := data["failed"].([]any); len(failed) != 0 {
+		t.Fatalf("unexpected failed items: %#v", failed)
+	}
+}
+
 func TestServeIMSecurityURLCheck(t *testing.T) {
 	ctrl := gomock.NewController(t)
 	defer ctrl.Finish()
@@ -4533,6 +4679,18 @@ func TestServeIMConversationContextAcceptsUserTopic(t *testing.T) {
 	}
 }
 
+func TestNormalizeP2PConversationTopicParsesTinodeTopic(t *testing.T) {
+	user := types.Uid(1)
+	peer := types.Uid(2)
+	topic := user.P2PName(peer)
+
+	normalized, gotPeer := normalizeP2PConversationTopic(topic, user)
+
+	if normalized != topic || gotPeer != peer {
+		t.Fatalf("normalize p2p: got topic=%q peer=%s, want topic=%q peer=%s", normalized, gotPeer.UserId(), topic, peer.UserId())
+	}
+}
+
 func TestServeIMP2PConversation(t *testing.T) {
 	ctrl := gomock.NewController(t)
 	defer ctrl.Finish()

server/internal_admin_id_test.go

@@ -0,0 +1,15 @@
+package main
+
+import "testing"
+
+func TestInternalAdminDBIDKeepsNumericID(t *testing.T) {
+	const rawID int64 = 1559676214946304000
+
+	id, ok := parseInternalAdminDBID("1559676214946304000")
+	if !ok {
+		t.Fatal("numeric admin id was rejected")
+	}
+	if id != rawID {
+		t.Fatalf("numeric admin id = %d, want %d", id, rawID)
+	}
+}

server/internal_admin_query_resources.go

@@ -0,0 +1,413 @@
+package main
+
+import (
+	"fmt"
+	"net/http"
+	"strconv"
+	"strings"
+
+	"github.com/tinode/chat/server/store"
+	"github.com/tinode/chat/server/store/types"
+)
+
+type internalQueryFilter struct {
+	Param  string
+	Column string
+	Op     string
+	Kind   string
+}
+
+type internalQuerySpec struct {
+	Select      string
+	From        string
+	DefaultSort string
+	Filters     []internalQueryFilter
+	DetailKey   string
+	ScopeColumn string
+}
+
+var internalGroupResourceSpecs = map[string]*internalQuerySpec{
+	"join-requests": {
+		Select: "SELECT id,request_id,topic,applicant_user_id,inviter_user_id,source,message,status,reject_reason,handled_by,handled_at,expires_at,version,created_at,updated_at",
+		From:   "FROM im_group_join_requests", DefaultSort: "ORDER BY id DESC", ScopeColumn: "topic",
+		Filters: []internalQueryFilter{{"status", "status", "=", "string"}, {"applicant_user_id", "applicant_user_id", "=", "uid"}},
+	},
+	"invitations": {
+		Select: "SELECT id,invitation_id,topic,inviter_user_id,target_user_id,message,status,result,join_request_id,expires_at,handled_at,created_at,updated_at",
+		From:   "FROM im_group_invitations", DefaultSort: "ORDER BY id DESC", ScopeColumn: "topic",
+		Filters: []internalQueryFilter{{"status", "status", "=", "string"}, {"target_user_id", "target_user_id", "=", "uid"}, {"inviter_user_id", "inviter_user_id", "=", "uid"}},
+	},
+	"announcements": {
+		Select: "SELECT id,announcement_id,topic,author_user_id,content,status,version,published_at,deleted_by,deleted_at,created_at,updated_at",
+		From:   "FROM im_group_announcements", DefaultSort: "ORDER BY id DESC", ScopeColumn: "topic",
+		Filters: []internalQueryFilter{{"status", "status", "=", "string"}},
+	},
+	"mutes": {
+		Select: "SELECT id,topic,user_id,muted_by,muted_at,expires_at,reason,status,revoked_by,revoked_at,version,created_at,updated_at",
+		From:   "FROM im_group_mutes", DefaultSort: "ORDER BY id DESC", ScopeColumn: "topic",
+		Filters: []internalQueryFilter{{"status", "status", "=", "string"}, {"user_id", "user_id", "=", "uid"}},
+	},
+	"admin-logs": {
+		Select: "SELECT id,log_id,topic,operator_id,target_user_id,action,object_type,object_id,before_data,after_data,request_id,idempotency_key,created_at",
+		From:   "FROM im_group_admin_logs", DefaultSort: "ORDER BY id DESC", ScopeColumn: "topic",
+		Filters: []internalQueryFilter{{"action", "action", "=", "string"}, {"operator_id", "operator_id", "=", "uid"}},
+	},
+	"files": {
+		Select: "SELECT id,group_file_id,topic,file_id,source_topic,source_seq,uploader_id,file_name,mime_type,file_type,file_size,status,deleted_by,deleted_at,created_at,updated_at",
+		From:   "FROM im_group_files", DefaultSort: "ORDER BY id DESC", ScopeColumn: "topic",
+		Filters: []internalQueryFilter{{"status", "status", "=", "string"}, {"file_type", "file_type", "=", "string"}, {"file_name", "file_name", "LIKE", "string"}, {"uploader_id", "uploader_id", "=", "uid"}},
+	},
+}
+
+var internalTenantResourceSpecs = map[string]*internalQuerySpec{
+	"audit-logs": {
+		Select: "SELECT id,request_id,actor_user_id,action,target_type,target_id,ip,user_agent,detail,created_at",
+		From:   "FROM im_audit_logs", DefaultSort: "ORDER BY id DESC", DetailKey: "id",
+		Filters: []internalQueryFilter{{"actor_user_id", "actor_user_id", "=", "uid"}, {"action", "action", "=", "string"}, {"target_type", "target_type", "=", "string"}, {"target_id", "target_id", "=", "string"}, {"created_at_start", "created_at", ">=", "time"}, {"created_at_end", "created_at", "<=", "time"}},
+	},
+	"realname-verifications": {
+		Select: "SELECT id,user_id,submission_id,status,masked_name AS real_name,masked_id_number AS id_card_no_masked,review_reason AS reject_reason,verified_at AS reviewed_at,created_at,updated_at",
+		From:   "FROM im_realname_verifications", DefaultSort: "ORDER BY id DESC", DetailKey: "id",
+		Filters: []internalQueryFilter{{"user_id", "user_id", "=", "uid"}, {"status", "status", "=", "string"}, {"real_name", "masked_name", "LIKE", "string"}, {"created_at_start", "created_at", ">=", "time"}, {"created_at_end", "created_at", "<=", "time"}},
+	},
+	"abuse-reports": {
+		Select: "SELECT id,report_id,reporter_id AS reporter_user_id,target_type,target_user_id,target_topic,target_seq,reason_code,description,evidence_file_ids,snapshot,status,created_at,updated_at",
+		From:   "FROM im_abuse_reports", DefaultSort: "ORDER BY id DESC", DetailKey: "id",
+		Filters: []internalQueryFilter{{"reporter_user_id", "reporter_id", "=", "uid"}, {"target_type", "target_type", "=", "string"}, {"status", "status", "=", "string"}, {"created_at_start", "created_at", ">=", "time"}, {"created_at_end", "created_at", "<=", "time"}},
+	},
+	"feedback-submissions": {
+		Select: "SELECT id,feedback_id,user_id,category,content,contact,attachment_file_ids AS file_ids,client_context,include_diagnostics,status,created_at,updated_at",
+		From:   "FROM im_feedback_submissions", DefaultSort: "ORDER BY id DESC", DetailKey: "id",
+		Filters: []internalQueryFilter{{"user_id", "user_id", "=", "uid"}, {"category", "category", "=", "string"}, {"status", "status", "=", "string"}, {"created_at_start", "created_at", ">=", "time"}, {"created_at_end", "created_at", "<=", "time"}},
+	},
+	"feedback-config": {
+		Select: "SELECT id,categories,max_length,attachment_policy,allow_contact,diagnostics_notice,created_at,updated_at",
+		From:   "FROM im_feedback_config", DefaultSort: "ORDER BY id DESC", DetailKey: "id",
+	},
+	"mailbox-config": {
+		Select: "SELECT id,enabled,allow_anonymous,required_fields,max_length,attachment_policy,notice_title,notice_content,notice_updated_at,created_at,updated_at",
+		From:   "FROM im_mailbox_config", DefaultSort: "ORDER BY id DESC", DetailKey: "id",
+	},
+	"abuse-reasons": {
+		Select: "SELECT id,target_type,reason_code,label,requires_description,allows_evidence,enabled,sort_order,created_at,updated_at",
+		From:   "FROM im_abuse_reasons", DefaultSort: "ORDER BY sort_order ASC,id ASC", DetailKey: "id",
+		Filters: []internalQueryFilter{{"target_type", "target_type", "=", "string"}},
+	},
+	"app-releases": {
+		Select: "SELECT id,platform,channel,latest_version,min_supported_version,update_type,release_notes,store_url,package_url,checksum,remind_after_seconds,published_at,created_at,updated_at",
+		From:   "FROM im_app_releases", DefaultSort: "ORDER BY id DESC", DetailKey: "id",
+	},
+	"workbench/apps": {
+		Select: "SELECT id,code AS app_code,name,icon_file_id,route,sort_order,unread_key,enabled,permission,created_at,updated_at",
+		From:   "FROM im_workbench_apps", DefaultSort: "ORDER BY sort_order ASC,id ASC", DetailKey: "id",
+	},
+}
+
+var internalOutboxResourceSpecs = map[string]*internalQuerySpec{
+	"business-events": {
+		Select: "SELECT id,event_id,target_type,target,actor_user_id,event_type,payload,status,attempts,next_retry_at,delivered_at,created_at,updated_at",
+		From:   "FROM im_business_events_outbox", DefaultSort: "ORDER BY id DESC", DetailKey: "id",
+		Filters: []internalQueryFilter{{"event_type", "event_type", "=", "string"}, {"status", "status", "=", "string"}, {"target_type", "target_type", "=", "string"}, {"target", "target", "=", "string"}, {"created_at_start", "created_at", ">=", "time"}, {"created_at_end", "created_at", "<=", "time"}},
+	},
+	"group-events": {
+		Select: "SELECT id,event_id,topic,actor_user_id,event_type,payload,status,attempts,next_retry_at,delivered_at,created_at,updated_at",
+		From:   "FROM im_group_events_outbox", DefaultSort: "ORDER BY id DESC", DetailKey: "id",
+		Filters: []internalQueryFilter{{"event_type", "event_type", "=", "string"}, {"status", "status", "=", "string"}, {"topic", "topic", "=", "string"}, {"created_at_start", "created_at", ">=", "time"}, {"created_at_end", "created_at", "<=", "time"}},
+	},
+}
+
+var internalOrgResourceSpecs = map[string]*internalQuerySpec{
+	"departments": {
+		Select: "SELECT id,department_id,parent_id,name,sort_order,state,created_at,updated_at",
+		From:   "FROM im_org_departments", DefaultSort: "ORDER BY sort_order ASC,id ASC",
+		Filters: []internalQueryFilter{{"parent_id", "parent_id", "=", "string"}, {"state", "state", "=", "string"}},
+	},
+	"members": {
+		Select: "SELECT id,department_id,user_id,display_name,title,mobile,email,employment_state,sort_order,created_at,updated_at",
+		From:   "FROM im_org_members", DefaultSort: "ORDER BY sort_order ASC,id ASC",
+		Filters: []internalQueryFilter{{"department_id", "department_id", "=", "string"}, {"user_id", "user_id", "=", "uid"}, {"employment_state", "employment_state", "=", "string"}},
+	},
+}
+
+func serveInternalAdminResourceList(wrt http.ResponseWriter, req *http.Request, tenantID types.TenantID, resource, scope string) {
+	spec, ok := internalAdminQuerySpec(resource)
+	if !ok {
+		writeIMError(wrt, req, http.StatusNotFound, 30004, imReasonMalformedRequest, "resource not found", nil)
+		return
+	}
+	query, countQuery, args, page, perPage, err := buildInternalAdminListQuery(req, spec, scope)
+	if err != nil {
+		writeIMError(wrt, req, http.StatusBadRequest, 30000, imReasonMalformedRequest, "malformed request", nil)
+		return
+	}
+	result, err := store.InternalAdminQueries.Query(tenantID, query, countQuery, args, page, perPage)
+	if err != nil {
+		writeInternalQueryError(wrt, req, err)
+		return
+	}
+	writeIMSuccess(wrt, req, map[string]any{
+		"items":    normalizeInternalAdminRowsForAPI(result.Items),
+		"total":    result.Total,
+		"page":     result.Page,
+		"per_page": result.PerPage,
+	})
+}
+
+func serveInternalAdminResourceDetail(wrt http.ResponseWriter, req *http.Request, tenantID types.TenantID, resource, id string) {
+	spec, ok := internalAdminQuerySpec(resource)
+	if !ok || spec.DetailKey == "" {
+		writeIMError(wrt, req, http.StatusNotFound, 30004, imReasonMalformedRequest, "resource not found", nil)
+		return
+	}
+	query := spec.Select + " " + spec.From + " WHERE " + spec.DetailKey + "=?"
+	row, err := store.InternalAdminQueries.QueryOne(tenantID, query, []any{id})
+	if err != nil {
+		writeInternalQueryError(wrt, req, err)
+		return
+	}
+	writeIMSuccess(wrt, req, normalizeInternalAdminRowForAPI(row))
+}
+
+func serveInternalDashboardSummary(wrt http.ResponseWriter, req *http.Request, tenantID types.TenantID) {
+	summary := map[string]any{}
+	queries := map[string]string{
+		"dead_kafka":        "SELECT COUNT(*) AS count FROM im_kafka_dead_letters WHERE status='dead'",
+		"pending_business":  "SELECT COUNT(*) AS count FROM im_business_events_outbox WHERE status='pending'",
+		"pending_group":     "SELECT COUNT(*) AS count FROM im_group_events_outbox WHERE status='pending'",
+		"pending_realname":  "SELECT COUNT(*) AS count FROM im_realname_verifications WHERE status='pending'",
+		"received_abuse":    "SELECT COUNT(*) AS count FROM im_abuse_reports WHERE status='received'",
+		"received_feedback": "SELECT COUNT(*) AS count FROM im_feedback_submissions WHERE status='received'",
+		"active_groups":     "SELECT COUNT(*) AS count FROM im_group_settings WHERE group_status='active'",
+		"suspended_users":   "SELECT COUNT(*) AS count FROM im_users WHERE state=10",
+	}
+	for key, query := range queries {
+		row, err := store.InternalAdminQueries.QueryOne(tenantID, query, nil)
+		if err != nil {
+			summary[key] = nil
+			continue
+		}
+		summary[key] = row["count"]
+	}
+	writeIMSuccess(wrt, req, summary)
+}
+
+func serveInternalAbuseTargetQuery(wrt http.ResponseWriter, req *http.Request, tenantID types.TenantID, reportID string) {
+	row, err := store.InternalAdminQueries.QueryOne(tenantID,
+		"SELECT id,report_id,target_type,target_user_id,target_topic,target_seq,snapshot FROM im_abuse_reports WHERE report_id=?",
+		[]any{strings.TrimSpace(reportID)})
+	if err != nil {
+		writeInternalQueryError(wrt, req, err)
+		return
+	}
+	data := normalizeInternalAdminRowForAPI(row)
+	switch fmt.Sprint(row["target_type"]) {
+	case "user":
+		if id, ok := numericID(row["target_user_id"]); ok && id > 0 {
+			if user, getErr := store.InternalAdminQueries.QueryOne(tenantID, usersInternalQuerySpec().Select+" "+usersInternalQuerySpec().From+" WHERE u.id=?", []any{id}); getErr == nil {
+				data["target"] = normalizeInternalAdminRowForAPI(user)
+			}
+		}
+	case "message":
+		topic := fmt.Sprint(row["target_topic"])
+		seq, _ := numericID(row["target_seq"])
+		if topic != "" && seq > 0 {
+			if msg, getErr := store.Messages.GetOne(tenantID, topic, seq); getErr == nil {
+				data["target"] = map[string]any{"topic": msg.Topic, "seq_id": msg.SeqId, "from": msg.From, "head": msg.Head, "content": msg.Content, "created_at": msg.CreatedAt}
+			}
+		}
+	case "group":
+		topic := fmt.Sprint(row["target_topic"])
+		if topic != "" {
+			if group, getErr := store.InternalAdminQueries.QueryOne(tenantID, groupsInternalQuerySpec().Select+" "+groupsInternalQuerySpec().From+" WHERE t.name=?", []any{topic}); getErr == nil {
+				data["target"] = normalizeInternalAdminRowForAPI(group)
+			}
+		}
+	}
+	writeIMSuccess(wrt, req, data)
+}
+
+func internalAdminQuerySpec(resource string) (*internalQuerySpec, bool) {
+	switch resource {
+	case "users":
+		return usersInternalQuerySpec(), true
+	case "user_devices":
+		return &internalQuerySpec{Select: "SELECT id,deviceid AS device_id,platform,lastseen AS last_seen,lang", From: "FROM im_devices", DefaultSort: "ORDER BY lastseen DESC", ScopeColumn: "userid", Filters: []internalQueryFilter{{"device_id", "deviceid", "LIKE", "string"}, {"platform", "platform", "=", "string"}}}, true
+	case "groups":
+		return groupsInternalQuerySpec(), true
+	case "file_links":
+		return &internalQuerySpec{Select: "SELECT id,topic,msgid AS seq_id,userid AS user_id,createdat AS created_at", From: "FROM im_filemsglinks", DefaultSort: "ORDER BY id DESC", ScopeColumn: "fileid"}, true
+	case "files":
+		return &internalQuerySpec{Select: "SELECT id,id AS file_id,userid AS user_id,mimetype AS mime_type,size AS file_size,location,etag,status,createdat AS created_at,updatedat AS updated_at", From: "FROM im_fileuploads", DefaultSort: "ORDER BY id DESC", DetailKey: "id", Filters: []internalQueryFilter{{"file_id", "id", "=", "int"}, {"user_id", "userid", "=", "uid"}, {"status", "status", "=", "int"}, {"mime_type", "mimetype", "LIKE", "string"}, {"created_at_start", "createdat", ">=", "time"}, {"created_at_end", "createdat", "<=", "time"}}}, true
+	}
+	if strings.HasPrefix(resource, "group_") {
+		spec := internalGroupResourceSpecs[strings.TrimPrefix(resource, "group_")]
+		return spec, spec != nil
+	}
+	if strings.HasPrefix(resource, "outbox_") {
+		spec := internalOutboxResourceSpecs[strings.TrimPrefix(resource, "outbox_")]
+		return spec, spec != nil
+	}
+	if strings.HasPrefix(resource, "org_") {
+		spec := internalOrgResourceSpecs[strings.TrimPrefix(resource, "org_")]
+		return spec, spec != nil
+	}
+	spec := internalTenantResourceSpecs[resource]
+	return spec, spec != nil
+}
+
+func usersInternalQuerySpec() *internalQuerySpec {
+	return &internalQuerySpec{
+		Select: `SELECT u.id AS user_id,u.username,u.phone,u.state,u.stateat AS state_at,u.access,u.lastseen AS last_seen,u.useragent AS user_agent,u.public,u.trusted,
+			p.nickname,p.public_account,p.gender,p.region,p.avatar_file_id,p.signature,sec.auth_version,sec.password_updated_at,u.createdat AS created_at,u.updatedat AS updated_at,
+			(SELECT JSON_ARRAYAGG(JSON_OBJECT('method',c.method,'value',c.value,'done',c.done,'updated_at',c.updatedat)) FROM im_credentials AS c WHERE c.userid=u.id AND c.deletedat IS NULL) AS credentials`,
+		From:        "FROM im_users AS u LEFT JOIN im_user_profiles AS p ON p.user_id=u.id LEFT JOIN im_user_security AS sec ON sec.user_id=u.id",
+		DefaultSort: "ORDER BY u.id DESC",
+		DetailKey:   "u.id",
+		Filters:     []internalQueryFilter{{"user_id", "u.id", "=", "uid"}, {"nickname", "p.nickname", "LIKE", "string"}, {"auth_name", "u.username", "LIKE", "string"}, {"state", "u.state", "=", "int"}, {"created_at_start", "u.createdat", ">=", "time"}, {"created_at_end", "u.createdat", "<=", "time"}, {"last_seen_start", "u.lastseen", ">=", "time"}, {"last_seen_end", "u.lastseen", "<=", "time"}},
+	}
+}
+
+func groupsInternalQuerySpec() *internalQuerySpec {
+	return &internalQuerySpec{
+		Select: `SELECT t.name AS topic,t.state,t.owner AS owner_id,t.seqid AS seq_id,t.delid AS del_id,t.subcnt AS sub_count,t.touchedat AS touched_at,t.createdat AS created_at,t.updatedat AS updated_at,t.public,t.trusted,t.access,t.tags,t.aux,
+			gs.name AS group_name,gs.avatar_file_id,gs.description,gs.group_status,gs.join_approval,gs.mute_all,gs.member_upload_files,gs.member_private_chat,gs.member_invite,gs.max_member_count,gs.max_admin_count,gs.member_count AS business_member_count,gs.version AS settings_version,gs.updated_at AS settings_updated_at`,
+		From:        "FROM im_topics AS t LEFT JOIN im_group_settings AS gs ON gs.topic=t.name",
+		DefaultSort: "ORDER BY t.id DESC",
+		DetailKey:   "t.name",
+		Filters:     []internalQueryFilter{{"topic", "t.name", "LIKE", "string"}, {"owner_id", "t.owner", "=", "uid"}, {"state", "t.state", "=", "int"}, {"group_status", "gs.group_status", "=", "string"}, {"created_at_start", "t.createdat", ">=", "time"}, {"created_at_end", "t.createdat", "<=", "time"}, {"touched_at_start", "t.touchedat", ">=", "time"}, {"touched_at_end", "t.touchedat", "<=", "time"}},
+	}
+}
+
+func buildInternalAdminListQuery(req *http.Request, spec *internalQuerySpec, scope string) (string, string, []any, int, int, error) {
+	page := parseIMPositiveInt(req.URL.Query().Get("page"), 1)
+	perPage := parseIMLimit(firstNonEmpty(req.URL.Query().Get("per_page"), req.URL.Query().Get("limit")), 20, 200)
+	where := []string{"1=1"}
+	args := []any{}
+	if spec.ScopeColumn != "" && strings.TrimSpace(scope) != "" {
+		value, err := internalAdminFilterValue("scope", scope, scopeKindForColumn(spec.ScopeColumn))
+		if err != nil {
+			return "", "", nil, 0, 0, err
+		}
+		where = append(where, spec.ScopeColumn+"=?")
+		args = append(args, value)
+	}
+	for _, filter := range spec.Filters {
+		raw := strings.TrimSpace(req.URL.Query().Get(filter.Param))
+		if raw == "" {
+			continue
+		}
+		value, err := internalAdminFilterValue(filter.Param, raw, filter.Kind)
+		if err != nil {
+			return "", "", nil, 0, 0, err
+		}
+		op := filter.Op
+		if op == "" {
+			op = "="
+		}
+		if strings.EqualFold(op, "LIKE") {
+			value = "%" + fmt.Sprint(value) + "%"
+		}
+		where = append(where, filter.Column+" "+op+" ?")
+		args = append(args, value)
+	}
+	whereSQL := strings.Join(where, " AND ")
+	query := spec.Select + " " + spec.From + " WHERE " + whereSQL + " " + spec.DefaultSort
+	countQuery := "SELECT COUNT(*) " + spec.From + " WHERE " + whereSQL
+	return query, countQuery, args, page, perPage, nil
+}
+
+func internalAdminFilterValue(_ string, raw, kind string) (any, error) {
+	switch kind {
+	case "uid":
+		id, ok := parseInternalAdminDBID(raw)
+		if !ok {
+			return nil, types.ErrMalformed
+		}
+		return id, nil
+	case "int":
+		return strconv.ParseInt(raw, 10, 64)
+	default:
+		return raw, nil
+	}
+}
+
+func scopeKindForColumn(column string) string {
+	switch column {
+	case "userid", "fileid":
+		return "uid"
+	default:
+		return "string"
+	}
+}
+
+func parseIMPositiveInt(raw string, fallback int) int {
+	value, err := strconv.Atoi(strings.TrimSpace(raw))
+	if err != nil || value <= 0 {
+		return fallback
+	}
+	return value
+}
+
+func normalizeInternalAdminRowsForAPI(rows []map[string]any) []map[string]any {
+	for i := range rows {
+		rows[i] = normalizeInternalAdminRowForAPI(rows[i])
+	}
+	return rows
+}
+
+func normalizeInternalAdminRowForAPI(row map[string]any) map[string]any {
+	return row
+}
+
+func parseInternalAdminDBID(raw string) (int64, bool) {
+	raw = strings.TrimSpace(raw)
+	if raw == "" {
+		return 0, false
+	}
+	if id, err := strconv.ParseInt(raw, 10, 64); err == nil && id > 0 {
+		return id, true
+	}
+	if uid := types.ParseUserId(raw); !uid.IsZero() {
+		return store.DecodeUid(uid), true
+	}
+	if uid := types.ParseUid(raw); !uid.IsZero() {
+		return store.DecodeUid(uid), true
+	}
+	return 0, false
+}
+
+func internalAdminUIDRaw(uid types.Uid) int64 {
+	if uid.IsZero() {
+		return 0
+	}
+	return store.DecodeUid(uid)
+}
+
+func internalAdminUIDStringRaw(raw string) any {
+	raw = strings.TrimSpace(raw)
+	if raw == "" {
+		return nil
+	}
+	if id, ok := parseInternalAdminDBID(raw); ok {
+		return id
+	}
+	return raw
+}
+
+func numericID(value any) (int64, bool) {
+	switch v := value.(type) {
+	case int64:
+		return v, true
+	case int:
+		return int64(v), true
+	case int32:
+		return int64(v), true
+	case uint64:
+		return int64(v), true
+	case string:
+		id, err := strconv.ParseInt(v, 10, 64)
+		return id, err == nil
+	default:
+		return 0, false
+	}
+}

server/internal_api.go

@@ -0,0 +1,594 @@
+package main
+
+import (
+	"bytes"
+	"crypto/hmac"
+	"crypto/sha256"
+	"encoding/base64"
+	"encoding/hex"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/tinode/chat/server/logs"
+	"github.com/tinode/chat/server/store"
+	"github.com/tinode/chat/server/store/types"
+)
+
+const (
+	internalCapabilityAdminQuery = "admin_query"
+
+	internalHeaderService   = "X-Service-Name"
+	internalHeaderTimestamp = "X-Timestamp"
+	internalHeaderNonce     = "X-Nonce"
+	internalHeaderSignature = "X-Signature"
+)
+
+type internalAPIConfig struct {
+	Enabled           bool                                `json:"enabled"`
+	TimeWindowSeconds int                                 `json:"time_window_seconds"`
+	Services          map[string]internalAPIServiceConfig `json:"services"`
+}
+
+type internalAPIServiceConfig struct {
+	Secret string   `json:"secret"`
+	Allow  []string `json:"allow"`
+}
+
+type internalAPIRuntime struct {
+	config internalAPIConfig
+	nonces sync.Map
+}
+
+var internalAPI internalAPIRuntime
+
+func initInternalAPI(raw json.RawMessage) error {
+	internalAPI = internalAPIRuntime{}
+	if len(raw) == 0 || string(raw) == "null" {
+		return nil
+	}
+	var cfg internalAPIConfig
+	if err := json.Unmarshal(raw, &cfg); err != nil {
+		return err
+	}
+	if cfg.TimeWindowSeconds <= 0 {
+		cfg.TimeWindowSeconds = 300
+	}
+	if cfg.Services == nil {
+		cfg.Services = map[string]internalAPIServiceConfig{}
+	}
+	internalAPI.config = cfg
+	if cfg.Enabled {
+		logs.Info.Printf("Internal API enabled for %d service(s)", len(cfg.Services))
+	}
+	return nil
+}
+
+func serveInternalAdminQuery(wrt http.ResponseWriter, req *http.Request, prefix string) {
+	if req.Method != http.MethodGet {
+		writeIMError(wrt, req, http.StatusMethodNotAllowed, 30002, imReasonMethodNotAllowed, "method not allowed", nil)
+		return
+	}
+	if !authorizeInternalAPI(wrt, req, internalCapabilityAdminQuery) {
+		return
+	}
+
+	tail := strings.Trim(strings.TrimPrefix(req.URL.Path, prefix), "/")
+	parts := splitInternalPath(tail)
+	if len(parts) < 3 || parts[0] != "tenants" {
+		writeIMError(wrt, req, http.StatusNotFound, 30004, imReasonMalformedRequest, "resource not found", nil)
+		return
+	}
+	tenant, ok := resolveInternalTenant(wrt, req, parts[1])
+	if !ok {
+		return
+	}
+
+	switch {
+	case len(parts) == 3 && parts[2] == "users":
+		serveInternalAdminResourceList(wrt, req, tenant.ID, "users", "")
+	case len(parts) == 4 && parts[2] == "users":
+		serveInternalUserQuery(wrt, req, tenant.ID, parts[3])
+	case len(parts) == 5 && parts[2] == "users" && parts[4] == "devices":
+		serveInternalAdminResourceList(wrt, req, tenant.ID, "user_devices", parts[3])
+	case len(parts) == 3 && parts[2] == "groups":
+		serveInternalAdminResourceList(wrt, req, tenant.ID, "groups", "")
+	case len(parts) == 4 && parts[2] == "groups":
+		serveInternalGroupQuery(wrt, req, tenant.ID, parts[3])
+	case len(parts) == 5 && parts[2] == "groups" && parts[4] == "members":
+		serveInternalGroupMembersQuery(wrt, req, tenant.ID, parts[3])
+	case len(parts) == 5 && parts[2] == "groups" && internalGroupResourceSpecs[parts[4]] != nil:
+		serveInternalAdminResourceList(wrt, req, tenant.ID, "group_"+parts[4], parts[3])
+	case len(parts) == 5 && parts[2] == "messages":
+		serveInternalMessageQuery(wrt, req, tenant.ID, parts[3], parts[4])
+	case len(parts) == 3 && parts[2] == "files":
+		serveInternalAdminResourceList(wrt, req, tenant.ID, "files", "")
+	case len(parts) == 4 && parts[2] == "files":
+		serveInternalFileQuery(wrt, req, tenant.ID, parts[3])
+	case len(parts) == 5 && parts[2] == "files" && parts[4] == "links":
+		serveInternalAdminResourceList(wrt, req, tenant.ID, "file_links", parts[3])
+	case len(parts) == 3 && internalTenantResourceSpecs[parts[2]] != nil:
+		serveInternalAdminResourceList(wrt, req, tenant.ID, parts[2], "")
+	case len(parts) == 4 && internalTenantResourceSpecs[parts[2]] != nil:
+		serveInternalAdminResourceDetail(wrt, req, tenant.ID, parts[2], parts[3])
+	case len(parts) == 4 && parts[2] == "reports" && parts[3] == "abuse":
+		writeIMError(wrt, req, http.StatusNotFound, 30004, imReasonMalformedRequest, "resource not found", nil)
+	case len(parts) == 6 && parts[2] == "reports" && parts[3] == "abuse" && parts[5] == "target":
+		serveInternalAbuseTargetQuery(wrt, req, tenant.ID, parts[4])
+	case len(parts) == 4 && parts[2] == "outbox" && internalOutboxResourceSpecs[parts[3]] != nil:
+		serveInternalAdminResourceList(wrt, req, tenant.ID, "outbox_"+parts[3], "")
+	case len(parts) == 5 && parts[2] == "outbox" && internalOutboxResourceSpecs[parts[3]] != nil:
+		serveInternalAdminResourceDetail(wrt, req, tenant.ID, "outbox_"+parts[3], parts[4])
+	case len(parts) == 4 && parts[2] == "org" && internalOrgResourceSpecs[parts[3]] != nil:
+		serveInternalAdminResourceList(wrt, req, tenant.ID, "org_"+parts[3], "")
+	case len(parts) == 4 && parts[2] == "workbench" && parts[3] == "apps":
+		serveInternalAdminResourceList(wrt, req, tenant.ID, "workbench/apps", "")
+	case len(parts) == 4 && parts[2] == "dashboard" && parts[3] == "summary":
+		serveInternalDashboardSummary(wrt, req, tenant.ID)
+	case len(parts) == 4 && parts[2] == "kafka" && parts[3] == "dead-letters":
+		serveInternalKafkaDeadLettersQuery(wrt, req, tenant.ID)
+	case len(parts) == 5 && parts[2] == "kafka" && parts[3] == "dead-letters":
+		serveInternalKafkaDeadLetterQuery(wrt, req, tenant.ID, parts[4])
+	default:
+		writeIMError(wrt, req, http.StatusNotFound, 30004, imReasonMalformedRequest, "resource not found", nil)
+	}
+}
+
+func authorizeInternalAPI(wrt http.ResponseWriter, req *http.Request, capability string) bool {
+	if !internalAPI.config.Enabled {
+		writeIMError(wrt, req, http.StatusNotFound, 30004, imReasonMalformedRequest, "resource not found", nil)
+		return false
+	}
+	serviceName := strings.TrimSpace(req.Header.Get(internalHeaderService))
+	timestampRaw := strings.TrimSpace(req.Header.Get(internalHeaderTimestamp))
+	nonce := strings.TrimSpace(req.Header.Get(internalHeaderNonce))
+	signature := strings.TrimSpace(req.Header.Get(internalHeaderSignature))
+	service, ok := internalAPI.config.Services[serviceName]
+	if !ok || service.Secret == "" || timestampRaw == "" || nonce == "" || signature == "" {
+		writeIMError(wrt, req, http.StatusUnauthorized, 30043, imReasonAuthRequired, "internal signature required", nil)
+		return false
+	}
+	if !internalServiceAllows(service, capability) {
+		writeIMError(wrt, req, http.StatusForbidden, 30005, imReasonPermissionDenied, "permission denied", nil)
+		return false
+	}
+	timestamp, err := strconv.ParseInt(timestampRaw, 10, 64)
+	if err != nil {
+		writeIMError(wrt, req, http.StatusUnauthorized, 30043, imReasonAuthRequired, "invalid internal timestamp", nil)
+		return false
+	}
+	window := time.Duration(internalAPI.config.TimeWindowSeconds) * time.Second
+	if delta := time.Since(time.Unix(timestamp, 0)); delta > window || delta < -window {
+		writeIMError(wrt, req, http.StatusUnauthorized, 30043, imReasonAuthRequired, "internal signature expired", nil)
+		return false
+	}
+	bodyHash, err := hashInternalRequestBody(req)
+	if err != nil {
+		writeIMError(wrt, req, http.StatusInternalServerError, 90000, imReasonInternal, "internal error", nil)
+		return false
+	}
+	signingText := strings.Join([]string{
+		req.Method,
+		req.URL.RequestURI(),
+		bodyHash,
+		timestampRaw,
+		nonce,
+	}, "\n")
+	if !verifyInternalSignature(service.Secret, signingText, signature) {
+		writeIMError(wrt, req, http.StatusUnauthorized, 30043, imReasonAuthRequired, "invalid internal signature", nil)
+		return false
+	}
+	if !claimInternalNonce(serviceName, nonce, time.Now().Add(window)) {
+		writeIMError(wrt, req, http.StatusUnauthorized, 30043, imReasonAuthRequired, "internal nonce replayed", nil)
+		return false
+	}
+	return true
+}
+
+func splitInternalPath(tail string) []string {
+	if tail == "" {
+		return nil
+	}
+	rawParts := strings.Split(tail, "/")
+	parts := make([]string, 0, len(rawParts))
+	for _, part := range rawParts {
+		if part == "" {
+			continue
+		}
+		decoded, err := url.PathUnescape(part)
+		if err != nil {
+			decoded = part
+		}
+		parts = append(parts, decoded)
+	}
+	return parts
+}
+
+func resolveInternalTenant(wrt http.ResponseWriter, req *http.Request, tenantCode string) (*types.Tenant, bool) {
+	if tenantCode == "" || store.Tenants == nil {
+		writeIMError(wrt, req, http.StatusBadRequest, 30000, imReasonMalformedRequest, "malformed request", map[string]any{"field": "tenant_code"})
+		return nil, false
+	}
+	tenant, err := store.Tenants.GetByCode(tenantCode)
+	if err != nil {
+		if errors.Is(err, types.ErrNotFound) {
+			writeIMError(wrt, req, http.StatusNotFound, 30010, imReasonTenantUnavailable, "tenant unavailable", nil)
+			return nil, false
+		}
+		writeIMError(wrt, req, http.StatusInternalServerError, 90000, imReasonInternal, "internal error", nil)
+		return nil, false
+	}
+	if !tenant.IsActive() {
+		writeIMError(wrt, req, http.StatusForbidden, 30010, imReasonTenantUnavailable, "tenant unavailable", nil)
+		return nil, false
+	}
+	return tenant, true
+}
+
+func serveInternalUserQuery(wrt http.ResponseWriter, req *http.Request, tenantID types.TenantID, userID string) {
+	userDBID, ok := parseInternalAdminDBID(userID)
+	if !ok {
+		writeIMError(wrt, req, http.StatusBadRequest, 30000, imReasonMalformedRequest, "malformed request", map[string]any{"field": "user_id"})
+		return
+	}
+	if spec, ok := internalAdminQuerySpec("users"); ok {
+		query := spec.Select + " " + spec.From + " WHERE u.id=?"
+		if row, err := store.InternalAdminQueries.QueryOne(tenantID, query, []any{userDBID}); err == nil {
+			writeIMSuccess(wrt, req, normalizeInternalAdminRowForAPI(row))
+			return
+		}
+	}
+	uid := store.EncodeUid(userDBID)
+	user, err := store.Users.Get(tenantID, uid)
+	if err != nil {
+		writeInternalQueryError(wrt, req, err)
+		return
+	}
+	writeIMSuccess(wrt, req, map[string]any{
+		"user_id":    userDBID,
+		"state":      user.State,
+		"state_at":   user.StateAt,
+		"username":   user.Username,
+		"phone":      user.Phone,
+		"public":     user.Public,
+		"trusted":    user.Trusted,
+		"last_seen":  user.LastSeen,
+		"user_agent": user.UserAgent,
+		"tags":       user.Tags,
+		"created_at": user.CreatedAt,
+		"updated_at": user.UpdatedAt,
+	})
+}
+
+func serveInternalGroupQuery(wrt http.ResponseWriter, req *http.Request, tenantID types.TenantID, topic string) {
+	if topic == "" || store.GroupBusiness == nil {
+		writeIMError(wrt, req, http.StatusBadRequest, 30000, imReasonMalformedRequest, "malformed request", map[string]any{"field": "topic"})
+		return
+	}
+	if spec, ok := internalAdminQuerySpec("groups"); ok {
+		query := spec.Select + " " + spec.From + " WHERE t.name=?"
+		if row, err := store.InternalAdminQueries.QueryOne(tenantID, query, []any{topic}); err == nil {
+			writeIMSuccess(wrt, req, normalizeInternalAdminRowForAPI(row))
+			return
+		}
+	}
+	settings, err := store.GroupBusiness.GetSettings(tenantID, topic)
+	if err != nil {
+		writeInternalQueryError(wrt, req, err)
+		return
+	}
+	writeIMSuccess(wrt, req, internalGroupSettingsPayload(settings))
+}
+
+func serveInternalGroupMembersQuery(wrt http.ResponseWriter, req *http.Request, tenantID types.TenantID, topic string) {
+	if topic == "" || store.GroupBusiness == nil {
+		writeIMError(wrt, req, http.StatusBadRequest, 30000, imReasonMalformedRequest, "malformed request", map[string]any{"field": "topic"})
+		return
+	}
+	spec := &internalQuerySpec{
+		Select: `SELECT gm.id,gm.topic,gm.user_id,u.state,u.public,p.nickname,p.public_account,s.modewant AS mode_want,s.modegiven AS mode_given,s.recvseqid AS recv_seq_id,s.readseqid AS read_seq_id,s.delid AS del_id,
+			gm.role,gm.member_status,gm.nickname AS group_nickname,gm.pinned,gm.mute_notifications,gm.join_source,gm.joined_by,gm.joined_at,gm.left_at,gm.removed_by,gm.version,gm.created_at,gm.updated_at`,
+		From:        "FROM im_group_members AS gm LEFT JOIN im_users AS u ON u.id=gm.user_id LEFT JOIN im_user_profiles AS p ON p.user_id=gm.user_id LEFT JOIN im_subscriptions AS s ON s.topic=gm.topic AND s.userid=gm.user_id",
+		DefaultSort: "ORDER BY gm.id DESC",
+		ScopeColumn: "gm.topic",
+		Filters: []internalQueryFilter{
+			{"user_id", "gm.user_id", "=", "uid"},
+			{"role", "gm.role", "=", "string"},
+			{"member_status", "gm.member_status", "=", "string"},
+			{"q", "gm.nickname", "LIKE", "string"},
+			{"eligible_role", "gm.role", "=", "string"},
+		},
+	}
+	query, countQuery, args, page, perPage, err := buildInternalAdminListQuery(req, spec, topic)
+	if err == nil {
+		if result, queryErr := store.InternalAdminQueries.Query(tenantID, query, countQuery, args, page, perPage); queryErr == nil {
+			writeIMSuccess(wrt, req, map[string]any{
+				"items":       normalizeInternalAdminRowsForAPI(result.Items),
+				"total":       result.Total,
+				"page":        result.Page,
+				"per_page":    result.PerPage,
+				"next_cursor": "",
+				"has_more":    false,
+			})
+			return
+		}
+	}
+	limit := parseIMLimit(req.URL.Query().Get("limit"), 50, 200)
+	members, nextCursor, err := store.GroupBusiness.ListMembers(tenantID, topic, types.GroupMemberListOptions{
+		Role:         strings.TrimSpace(req.URL.Query().Get("role")),
+		EligibleRole: strings.TrimSpace(req.URL.Query().Get("eligible_role")),
+		Query:        strings.TrimSpace(req.URL.Query().Get("q")),
+		Cursor:       strings.TrimSpace(req.URL.Query().Get("cursor")),
+		Limit:        limit,
+	})
+	if err != nil {
+		writeInternalQueryError(wrt, req, err)
+		return
+	}
+	items := make([]any, 0, len(members))
+	for _, member := range members {
+		items = append(items, map[string]any{
+			"user_id":            internalAdminUIDRaw(member.UserID),
+			"role":               member.Role,
+			"member_status":      member.MemberStatus,
+			"nickname":           member.Nickname,
+			"pinned":             member.Pinned,
+			"mute_notifications": member.MuteNotifications,
+			"join_source":        member.JoinSource,
+			"joined_by":          internalAdminUIDRaw(member.JoinedBy),
+			"joined_at":          member.JoinedAt,
+			"left_at":            member.LeftAt,
+			"removed_by":         internalAdminUIDRaw(member.RemovedBy),
+			"version":            member.Version,
+			"created_at":         member.CreatedAt,
+			"updated_at":         member.UpdatedAt,
+		})
+	}
+	writeIMSuccess(wrt, req, map[string]any{"items": items, "next_cursor": nextCursor})
+}
+
+func serveInternalMessageQuery(wrt http.ResponseWriter, req *http.Request, tenantID types.TenantID, topic, seqRaw string) {
+	seq, err := strconv.ParseInt(seqRaw, 10, 64)
+	if topic == "" || err != nil || seq <= 0 {
+		writeIMError(wrt, req, http.StatusBadRequest, 30000, imReasonMalformedRequest, "malformed request", nil)
+		return
+	}
+	msg, err := store.Messages.GetOne(tenantID, topic, seq)
+	if err != nil {
+		writeInternalQueryError(wrt, req, err)
+		return
+	}
+	writeIMSuccess(wrt, req, map[string]any{
+		"topic":         msg.Topic,
+		"seq_id":        msg.SeqId,
+		"from":          internalAdminUIDStringRaw(msg.From),
+		"client_msg_id": msg.ClientMsgID,
+		"version":       msg.Version,
+		"head":          msg.Head,
+		"content":       msg.Content,
+		"deleted_at":    msg.DeletedAt,
+		"recalled_at":   msg.RecalledAt,
+		"recalled_by":   internalAdminUIDStringRaw(msg.RecalledBy),
+		"created_at":    msg.CreatedAt,
+		"updated_at":    msg.UpdatedAt,
+	})
+}
+
+func serveInternalFileQuery(wrt http.ResponseWriter, req *http.Request, tenantID types.TenantID, fileID string) {
+	if fileID == "" || store.Files == nil {
+		writeIMError(wrt, req, http.StatusBadRequest, 30000, imReasonMalformedRequest, "malformed request", map[string]any{"field": "file_id"})
+		return
+	}
+	file, err := store.Files.Get(tenantID, fileID)
+	if err != nil {
+		writeInternalQueryError(wrt, req, err)
+		return
+	}
+	writeIMSuccess(wrt, req, map[string]any{
+		"file_id":    file.Id,
+		"status":     file.Status,
+		"user_id":    internalAdminUIDStringRaw(file.User),
+		"mime_type":  file.MimeType,
+		"size":       file.Size,
+		"location":   file.Location,
+		"etag":       file.ETag,
+		"created_at": file.CreatedAt,
+		"updated_at": file.UpdatedAt,
+	})
+}
+
+func serveInternalKafkaDeadLettersQuery(wrt http.ResponseWriter, req *http.Request, tenantID types.TenantID) {
+	if store.HTTPInfrastructure == nil {
+		writeIMError(wrt, req, http.StatusInternalServerError, 90000, imReasonInternal, "internal error", nil)
+		return
+	}
+	limit := parseIMLimit(req.URL.Query().Get("limit"), 50, 200)
+	page, err := store.HTTPInfrastructure.ListKafkaDeadLetters(tenantID, types.KafkaDeadLetterListOptions{
+		Status:    strings.TrimSpace(req.URL.Query().Get("status")),
+		BizType:   strings.TrimSpace(req.URL.Query().Get("biz_type")),
+		BizID:     strings.TrimSpace(req.URL.Query().Get("biz_id")),
+		EventType: strings.TrimSpace(req.URL.Query().Get("event_type")),
+		Cursor:    strings.TrimSpace(req.URL.Query().Get("cursor")),
+		Limit:     limit,
+	})
+	if err != nil {
+		writeInternalQueryError(wrt, req, err)
+		return
+	}
+	items := make([]any, 0, len(page.Items))
+	for _, dead := range page.Items {
+		items = append(items, internalKafkaDeadLetterPayload(&dead, false))
+	}
+	writeIMSuccess(wrt, req, map[string]any{
+		"items":       items,
+		"next_cursor": page.NextCursor,
+		"has_more":    page.NextCursor != "",
+	})
+}
+
+func serveInternalKafkaDeadLetterQuery(wrt http.ResponseWriter, req *http.Request, tenantID types.TenantID, deadID string) {
+	if store.HTTPInfrastructure == nil || strings.TrimSpace(deadID) == "" {
+		writeIMError(wrt, req, http.StatusBadRequest, 30000, imReasonMalformedRequest, "malformed request", map[string]any{"field": "dead_id"})
+		return
+	}
+	dead, err := store.HTTPInfrastructure.GetKafkaDeadLetter(tenantID, deadID)
+	if err != nil {
+		writeInternalQueryError(wrt, req, err)
+		return
+	}
+	writeIMSuccess(wrt, req, internalKafkaDeadLetterPayload(dead, true))
+}
+
+func internalGroupSettingsPayload(settings *types.GroupSettings) map[string]any {
+	if settings == nil {
+		return nil
+	}
+	return map[string]any{
+		"topic":               settings.Topic,
+		"owner_user_id":       internalAdminUIDRaw(settings.OwnerUserID),
+		"name":                settings.Name,
+		"avatar_file_id":      settings.AvatarFileID,
+		"description":         settings.Description,
+		"group_status":        settings.GroupStatus,
+		"join_approval":       settings.JoinApproval,
+		"mute_all":            settings.MuteAll,
+		"member_upload_files": settings.MemberUploadFiles,
+		"member_private_chat": settings.MemberPrivateChat,
+		"member_invite":       settings.MemberInvite,
+		"max_member_count":    settings.MaxMemberCount,
+		"max_admin_count":     settings.MaxAdminCount,
+		"member_count":        settings.MemberCount,
+		"version":             settings.Version,
+		"created_by":          internalAdminUIDRaw(settings.CreatedBy),
+		"dissolved_by":        internalAdminUIDRaw(settings.DissolvedBy),
+		"dissolved_at":        settings.DissolvedAt,
+		"created_at":          settings.CreatedAt,
+		"updated_at":          settings.UpdatedAt,
+	}
+}
+
+func internalKafkaDeadLetterPayload(dead *types.KafkaDeadLetter, includeEnvelope bool) map[string]any {
+	if dead == nil {
+		return nil
+	}
+	payload := map[string]any{
+		"id":              dead.ID,
+		"dead_id":         dead.DeadID,
+		"event_id":        dead.EventID,
+		"parent_event_id": dead.ParentEventID,
+		"source_topic":    dead.SourceTopic,
+		"consumer_group":  dead.ConsumerGroup,
+		"event_type":      dead.EventType,
+		"biz_type":        dead.BizType,
+		"biz_id":          dead.BizID,
+		"retry_count":     dead.RetryCount,
+		"last_reason":     dead.LastReason,
+		"last_error":      dead.LastError,
+		"status":          dead.Status,
+		"replay_count":    dead.ReplayCount,
+		"created_at":      dead.CreatedAt,
+		"updated_at":      dead.UpdatedAt,
+	}
+	if includeEnvelope {
+		var envelope any
+		if len(dead.Envelope) > 0 && json.Unmarshal(dead.Envelope, &envelope) == nil {
+			payload["envelope"] = envelope
+		} else {
+			payload["envelope_raw"] = string(dead.Envelope)
+		}
+	}
+	return payload
+}
+
+func writeInternalQueryError(wrt http.ResponseWriter, req *http.Request, err error) {
+	switch {
+	case errors.Is(err, types.ErrUserNotFound), errors.Is(err, types.ErrTopicNotFound), errors.Is(err, types.ErrNotFound):
+		writeIMError(wrt, req, http.StatusNotFound, 30004, imReasonMalformedRequest, "resource not found", nil)
+	case errors.Is(err, types.ErrMalformed):
+		writeIMError(wrt, req, http.StatusBadRequest, 30000, imReasonMalformedRequest, "malformed request", nil)
+	case errors.Is(err, types.ErrPermissionDenied):
+		writeIMError(wrt, req, http.StatusForbidden, 30005, imReasonPermissionDenied, "permission denied", nil)
+	default:
+		writeIMError(wrt, req, http.StatusInternalServerError, 90000, imReasonInternal, "internal error", nil)
+	}
+}
+
+func parseInternalUid(raw string) types.Uid {
+	if uid := types.ParseUserId(raw); !uid.IsZero() {
+		return uid
+	}
+	id, ok := parseInternalAdminDBID(raw)
+	if !ok {
+		return types.ZeroUid
+	}
+	return store.EncodeUid(id)
+}
+
+func internalServiceAllows(service internalAPIServiceConfig, capability string) bool {
+	for _, allowed := range service.Allow {
+		allowed = strings.TrimSpace(allowed)
+		if allowed == "*" || allowed == capability {
+			return true
+		}
+	}
+	return false
+}
+
+func hashInternalRequestBody(req *http.Request) (string, error) {
+	if req.Body == nil {
+		sum := sha256.Sum256(nil)
+		return hex.EncodeToString(sum[:]), nil
+	}
+	body, err := io.ReadAll(io.LimitReader(req.Body, 1<<20))
+	if err != nil {
+		return "", err
+	}
+	if err := req.Body.Close(); err != nil {
+		return "", err
+	}
+	req.Body = io.NopCloser(bytes.NewReader(body))
+	sum := sha256.Sum256(body)
+	return hex.EncodeToString(sum[:]), nil
+}
+
+func verifyInternalSignature(secret, signingText, signature string) bool {
+	signature = strings.TrimPrefix(signature, "sha256=")
+	mac := hmac.New(sha256.New, []byte(secret))
+	_, _ = mac.Write([]byte(signingText))
+	expected := mac.Sum(nil)
+	if decoded, err := hex.DecodeString(signature); err == nil {
+		return hmac.Equal(decoded, expected)
+	}
+	if decoded, err := base64.StdEncoding.DecodeString(signature); err == nil {
+		return hmac.Equal(decoded, expected)
+	}
+	return false
+}
+
+func claimInternalNonce(serviceName, nonce string, expiresAt time.Time) bool {
+	ttl := time.Until(expiresAt)
+	if ok, enabled, err := store.ClaimShortState("internal_api_nonce", fmt.Sprintf("%s:%s", serviceName, nonce), ttl); enabled {
+		if err != nil {
+			logs.Warn.Println("internal api: failed to claim nonce in redis", err)
+			return false
+		}
+		return ok
+	}
+	now := time.Now()
+	internalAPI.nonces.Range(func(key, value any) bool {
+		if expires, ok := value.(time.Time); ok && expires.Before(now) {
+			internalAPI.nonces.Delete(key)
+		}
+		return true
+	})
+	nonceKey := fmt.Sprintf("%s:%s", serviceName, nonce)
+	_, loaded := internalAPI.nonces.LoadOrStore(nonceKey, expiresAt)
+	return !loaded
+}

server/kafka_admin_commands.go

@@ -0,0 +1,601 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/segmentio/kafka-go"
+	"github.com/tinode/chat/server/logs"
+	"github.com/tinode/chat/server/store"
+	"github.com/tinode/chat/server/store/types"
+)
+
+const (
+	adminCommandGroupProfileUpdate = "group.profile.update"
+	adminCommandGroupGovernUpdate  = "group.governance.update"
+	adminCommandGroupAdminSet      = "group.admin.set"
+	adminCommandGroupAdminRemove   = "group.admin.remove"
+	adminCommandGroupMemberMute    = "group.member.mute"
+	adminCommandGroupMemberUnmute  = "group.member.unmute"
+	adminCommandGroupMemberRemove  = "group.member.remove"
+	adminCommandGroupOwnerTransfer = "group.owner.transfer"
+	adminCommandUserBan            = "user.ban"
+	adminCommandUserUnban          = "user.unban"
+	adminCommandUserFreeze         = "user.freeze"
+	adminCommandUserUnfreeze       = "user.unfreeze"
+	adminCommandUserDeviceKick     = "user.device.kick"
+	adminCommandUserLogoutAll      = "user.logout_all"
+	adminCommandUserCancel         = "user.account.cancel"
+	adminCommandUserDelete         = "user.account.delete"
+	adminCommandKafkaDeadReplay    = "kafka.dead_letter.replay"
+	adminCommandKafkaDeadIgnore    = "kafka.dead_letter.ignore"
+	adminCommandKafkaDeadResolve   = "kafka.dead_letter.resolve"
+)
+
+type adminCommandPayload struct {
+	CommandID   string               `json:"command_id"`
+	ActorUserID string               `json:"actor_user_id"`
+	Topic       string               `json:"topic"`
+	UserID      string               `json:"user_id"`
+	UserIDs     []string             `json:"user_ids"`
+	TargetID    string               `json:"target_user_id"`
+	DeviceID    string               `json:"device_id"`
+	Version     int                  `json:"version"`
+	Reason      string               `json:"reason"`
+	DeadID      string               `json:"dead_id"`
+	Note        string               `json:"note"`
+	Hard        bool                 `json:"hard"`
+	ExpiresAt   *time.Time           `json:"expires_at"`
+	Profile     groupProfilePatch    `json:"profile"`
+	Governance  groupGovernancePatch `json:"governance"`
+}
+
+type groupProfilePatch struct {
+	Name         *string `json:"name"`
+	AvatarFileID *string `json:"avatar_file_id"`
+	Description  *string `json:"description"`
+}
+
+type groupGovernancePatch struct {
+	JoinApproval      *bool `json:"join_approval"`
+	MuteAll           *bool `json:"mute_all"`
+	MemberUploadFiles *bool `json:"member_upload_files"`
+	MemberPrivateChat *bool `json:"member_private_chat"`
+	MemberInvite      *bool `json:"member_invite"`
+}
+
+type adminCommandResultPayload struct {
+	CommandID      string         `json:"command_id"`
+	CommandEventID string         `json:"command_event_id"`
+	Status         string         `json:"status"`
+	Reason         string         `json:"reason,omitempty"`
+	Message        string         `json:"message,omitempty"`
+	Result         map[string]any `json:"result,omitempty"`
+}
+
+func startKafkaAdminCommandConsumer() {
+	startKafkaConsumer("admin_command", handleKafkaAdminCommand)
+}
+
+func handleKafkaAdminCommand(ctx context.Context, envelope kafkaEnvelope, _ kafka.Message) error {
+	if strings.TrimSpace(envelope.EventID) == "" {
+		return publishKafkaAdminCommandResult(ctx, envelope, "", "failed", "MALFORMED_EVENT", "missing event_id", nil)
+	}
+	if strings.TrimSpace(envelope.EventType) == "" {
+		return publishKafkaAdminCommandResult(ctx, envelope, "", "failed", "MALFORMED_EVENT", "missing event_type", nil)
+	}
+	var payload adminCommandPayload
+	if len(envelope.Payload) > 0 {
+		if err := json.Unmarshal(envelope.Payload, &payload); err != nil {
+			return publishKafkaAdminCommandResult(ctx, envelope, "", "failed", "MALFORMED_PAYLOAD", err.Error(), nil)
+		}
+	}
+	if payload.CommandID == "" {
+		payload.CommandID = envelope.EventID
+	}
+
+	tenantID, tenantErr := kafkaTenantID(envelope.TenantCode)
+	if tenantErr != nil {
+		return publishKafkaAdminCommandResult(ctx, envelope, payload.CommandID, "failed", kafkaCommandErrorReason(tenantErr), tenantErr.Error(), nil)
+	}
+	consume, claimed, beginErr := beginKafkaConsumeForTopic(tenantID, envelope, "admin_command")
+	if beginErr != nil {
+		return beginErr
+	}
+	if !claimed {
+		logs.Info.Printf("kafka admin command skipped: event=%s group=%s status=%s", envelope.EventID, kafkaBus.config.ConsumerGroup, consume.Status)
+		return nil
+	}
+
+	result, execErr := executeKafkaAdminCommand(tenantID, envelope, payload)
+	if execErr != nil {
+		reason := kafkaCommandErrorReason(execErr)
+		failedRecord, markErr := store.HTTPInfrastructure.MarkKafkaConsumeFailed(tenantID, envelope.EventID, kafkaBus.config.ConsumerGroup, reason, execErr.Error(), time.Now().UTC().Round(time.Millisecond))
+		if markErr != nil {
+			return markErr
+		}
+		if shouldDeadLetterKafkaCommand(reason, failedRecord.RetryCount) {
+			if deadErr := markKafkaDeadLetter(ctx, tenantID, envelope, failedRecord, reason, execErr.Error()); deadErr != nil {
+				return deadErr
+			}
+			if err := publishKafkaAdminCommandResult(ctx, envelope, payload.CommandID, "dead", reason, execErr.Error(), result); err != nil {
+				return err
+			}
+			logs.Warn.Printf("kafka admin command dead: event=%s type=%s reason=%s err=%v", envelope.EventID, envelope.EventType, reason, execErr)
+			return nil
+		}
+		if kafkaCommandReasonRetryable(reason) {
+			sleepKafkaRetryBackoff(failedRecord.RetryCount)
+			return execErr
+		}
+		if err := publishKafkaAdminCommandResult(ctx, envelope, payload.CommandID, "failed", reason, execErr.Error(), result); err != nil {
+			return err
+		}
+		logs.Warn.Printf("kafka admin command failed: event=%s type=%s reason=%s err=%v", envelope.EventID, envelope.EventType, reason, execErr)
+		return nil
+	}
+	if err := store.HTTPInfrastructure.MarkKafkaConsumeSucceeded(tenantID, envelope.EventID, kafkaBus.config.ConsumerGroup, time.Now().UTC().Round(time.Millisecond)); err != nil {
+		return err
+	}
+	if err := publishKafkaAdminCommandResult(ctx, envelope, payload.CommandID, "succeeded", "", "", result); err != nil {
+		return err
+	}
+	if err := enqueueKafkaAuditEvent(ctx, tenantID, "admin", parseKafkaCommandUser(payload.ActorUserID),
+		envelope.EventType, "admin_command", payload.CommandID, "succeeded", "", map[string]any{
+			"topic":    payload.Topic,
+			"biz_id":   envelope.BizID,
+			"event_id": envelope.EventID,
+			"result":   result,
+		}); err != nil {
+		logs.Warn.Printf("kafka admin command audit enqueue failed: event=%s err=%v", envelope.EventID, err)
+	}
+	return nil
+}
+
+func executeKafkaAdminCommand(tenantID types.TenantID, envelope kafkaEnvelope, payload adminCommandPayload) (map[string]any, error) {
+	actor := parseKafkaCommandUser(payload.ActorUserID)
+	if actor.IsZero() {
+		return nil, types.ErrMalformed
+	}
+	switch strings.TrimSpace(envelope.EventType) {
+	case adminCommandKafkaDeadReplay:
+		dead, err := replayKafkaDeadLetter(context.Background(), tenantID, payload.DeadID, actor, firstNonEmpty(payload.Note, payload.Reason))
+		if err != nil {
+			return nil, err
+		}
+		return kafkaDeadLetterResult(dead), nil
+	case adminCommandKafkaDeadIgnore:
+		if err := resolveKafkaDeadLetter(tenantID, payload.DeadID, types.KafkaDeadStatusIgnored, actor, firstNonEmpty(payload.Note, payload.Reason)); err != nil {
+			return nil, err
+		}
+		return map[string]any{"dead_id": payload.DeadID, "status": types.KafkaDeadStatusIgnored}, nil
+	case adminCommandKafkaDeadResolve:
+		if err := resolveKafkaDeadLetter(tenantID, payload.DeadID, types.KafkaDeadStatusResolved, actor, firstNonEmpty(payload.Note, payload.Reason)); err != nil {
+			return nil, err
+		}
+		return map[string]any{"dead_id": payload.DeadID, "status": types.KafkaDeadStatusResolved}, nil
+	case adminCommandUserBan, adminCommandUserUnban, adminCommandUserFreeze, adminCommandUserUnfreeze,
+		adminCommandUserDeviceKick, adminCommandUserLogoutAll, adminCommandUserCancel, adminCommandUserDelete:
+		return executeKafkaUserGovernanceCommand(tenantID, envelope, payload, actor)
+	}
+	if store.GroupBusiness == nil {
+		return nil, errors.New("group business store is not initialized")
+	}
+	topic := strings.TrimSpace(payload.Topic)
+	if topic == "" {
+		return nil, types.ErrMalformed
+	}
+	now := time.Now().UTC().Round(time.Millisecond)
+
+	switch strings.TrimSpace(envelope.EventType) {
+	case adminCommandGroupProfileUpdate:
+		settings, err := store.GroupBusiness.UpdateSettings(tenantID, topic, types.GroupSettingsPatch{
+			Name:         payload.Profile.Name,
+			AvatarFileID: payload.Profile.AvatarFileID,
+			Description:  payload.Profile.Description,
+			Version:      payload.Version,
+		})
+		if err != nil {
+			return nil, err
+		}
+		dispatchIMGroupEvents(tenantID)
+		return map[string]any{"topic": topic, "version": settings.Version, "updated_at": settings.UpdatedAt}, nil
+
+	case adminCommandGroupGovernUpdate:
+		settings, err := store.GroupBusiness.UpdateSettings(tenantID, topic, types.GroupSettingsPatch{
+			JoinApproval:      payload.Governance.JoinApproval,
+			MuteAll:           payload.Governance.MuteAll,
+			MemberUploadFiles: payload.Governance.MemberUploadFiles,
+			MemberPrivateChat: payload.Governance.MemberPrivateChat,
+			MemberInvite:      payload.Governance.MemberInvite,
+			Version:           payload.Version,
+		})
+		if err != nil {
+			return nil, err
+		}
+		dispatchIMGroupEvents(tenantID)
+		return map[string]any{"topic": topic, "version": settings.Version, "updated_at": settings.UpdatedAt}, nil
+
+	case adminCommandGroupAdminSet:
+		users := payload.UserIDs
+		if len(users) == 0 && payload.UserID != "" {
+			users = []string{payload.UserID}
+		}
+		if len(users) == 0 {
+			return nil, types.ErrMalformed
+		}
+		succeeded := make([]map[string]any, 0, len(users))
+		for _, raw := range users {
+			target := parseKafkaCommandUser(raw)
+			if target.IsZero() {
+				return map[string]any{"succeeded": succeeded}, types.ErrMalformed
+			}
+			if err := store.GroupBusiness.UpdateRole(tenantID, topic, actor, target,
+				types.GroupMemberRoleAdmin, types.ModeCAdmin|types.ModeCPublic, now); err != nil {
+				return map[string]any{"succeeded": succeeded, "failed_user_id": raw}, err
+			}
+			succeeded = append(succeeded, map[string]any{"user_id": internalAdminUIDRaw(target), "role": types.GroupMemberRoleAdmin})
+		}
+		dispatchIMGroupEvents(tenantID)
+		return map[string]any{"topic": topic, "succeeded": succeeded}, nil
+
+	case adminCommandGroupAdminRemove:
+		target := parseKafkaCommandUser(firstNonEmpty(payload.UserID, payload.TargetID))
+		if target.IsZero() {
+			return nil, types.ErrMalformed
+		}
+		if err := store.GroupBusiness.UpdateRole(tenantID, topic, actor, target,
+			types.GroupMemberRoleMember, types.ModeCPublic, now); err != nil {
+			return nil, err
+		}
+		dispatchIMGroupEvents(tenantID)
+		return map[string]any{"topic": topic, "user_id": internalAdminUIDRaw(target), "role": types.GroupMemberRoleMember}, nil
+
+	case adminCommandGroupMemberMute:
+		target := parseKafkaCommandUser(firstNonEmpty(payload.UserID, payload.TargetID))
+		if target.IsZero() {
+			return nil, types.ErrMalformed
+		}
+		mute, err := store.GroupBusiness.UpsertMute(tenantID, &types.GroupMute{
+			Topic:     topic,
+			UserID:    target,
+			MutedBy:   actor,
+			MutedAt:   now,
+			ExpiresAt: payload.ExpiresAt,
+			Reason:    strings.TrimSpace(payload.Reason),
+			Status:    types.GroupMuteStatusActive,
+		})
+		if err != nil {
+			return nil, err
+		}
+		dispatchIMGroupEvents(tenantID)
+		return map[string]any{"topic": topic, "user_id": internalAdminUIDRaw(target), "muted": true, "version": mute.Version}, nil
+
+	case adminCommandGroupMemberUnmute:
+		target := parseKafkaCommandUser(firstNonEmpty(payload.UserID, payload.TargetID))
+		if target.IsZero() {
+			return nil, types.ErrMalformed
+		}
+		if err := store.GroupBusiness.DeleteMute(tenantID, topic, target, actor, now); err != nil {
+			return nil, err
+		}
+		dispatchIMGroupEvents(tenantID)
+		return map[string]any{"topic": topic, "user_id": internalAdminUIDRaw(target), "muted": false}, nil
+
+	case adminCommandGroupMemberRemove:
+		target := parseKafkaCommandUser(firstNonEmpty(payload.UserID, payload.TargetID))
+		if target.IsZero() {
+			return nil, types.ErrMalformed
+		}
+		if err := store.GroupBusiness.Leave(tenantID, topic, target, now); err != nil {
+			return nil, err
+		}
+		dispatchIMGroupEvents(tenantID)
+		return map[string]any{"topic": topic, "user_id": internalAdminUIDRaw(target), "status": "left"}, nil
+
+	case adminCommandGroupOwnerTransfer:
+		target := parseKafkaCommandUser(firstNonEmpty(payload.TargetID, payload.UserID))
+		if target.IsZero() {
+			return nil, types.ErrMalformed
+		}
+		if err := store.GroupBusiness.TransferOwner(tenantID, topic, actor, target, payload.Version, now); err != nil {
+			return nil, err
+		}
+		dispatchIMGroupEvents(tenantID)
+		return map[string]any{"topic": topic, "owner": internalAdminUIDRaw(target), "previous_owner": internalAdminUIDRaw(actor)}, nil
+	}
+	return nil, errors.New("unsupported admin command: " + envelope.EventType)
+}
+
+func executeKafkaUserGovernanceCommand(tenantID types.TenantID, envelope kafkaEnvelope, payload adminCommandPayload, actor types.Uid) (map[string]any, error) {
+	target := parseKafkaCommandUser(firstNonEmpty(payload.TargetID, payload.UserID))
+	if target.IsZero() {
+		return nil, types.ErrMalformed
+	}
+	eventType := strings.TrimSpace(envelope.EventType)
+	now := time.Now().UTC().Round(time.Millisecond)
+	result := map[string]any{
+		"user_id": internalAdminUIDRaw(target),
+		"action":  eventType,
+	}
+
+	switch eventType {
+	case adminCommandUserBan, adminCommandUserFreeze:
+		if err := store.Users.UpdateState(tenantID, target, types.StateSuspended); err != nil {
+			return nil, err
+		}
+		evictUserSessions(target)
+		notifyUserStateChanged(tenantID, target, types.StateSuspended)
+		result["state"] = types.StateSuspended.String()
+		result["evicted"] = true
+		return result, nil
+	case adminCommandUserUnban, adminCommandUserUnfreeze:
+		if err := store.Users.UpdateState(tenantID, target, types.StateOK); err != nil {
+			return nil, err
+		}
+		notifyUserStateChanged(tenantID, target, types.StateOK)
+		result["state"] = types.StateOK.String()
+		return result, nil
+	case adminCommandUserDeviceKick:
+		deviceID := strings.TrimSpace(payload.DeviceID)
+		if deviceID == "" {
+			return nil, types.ErrMalformed
+		}
+		evicted := 0
+		if globals.sessionStore != nil {
+			evicted = globals.sessionStore.EvictUserDevice(target, deviceID)
+		}
+		if store.Devices != nil {
+			if err := store.Devices.Delete(tenantID, target, deviceID); err != nil && !errors.Is(err, types.ErrNotFound) {
+				return nil, err
+			}
+		}
+		result["device_id"] = deviceID
+		result["evicted_sessions"] = evicted
+		return result, nil
+	case adminCommandUserLogoutAll:
+		evictUserSessions(target)
+		result["evicted"] = true
+		return result, nil
+	case adminCommandUserCancel, adminCommandUserDelete:
+		hard := payload.Hard || eventType == adminCommandUserDelete
+		if err := deleteUserByAdminCommand(tenantID, target, hard, now); err != nil {
+			return nil, err
+		}
+		result["state"] = types.StateDeleted.String()
+		result["hard"] = hard
+		return result, nil
+	}
+	return nil, errors.New("unsupported user governance command: " + eventType)
+}
+
+func evictUserSessions(uid types.Uid) {
+	if globals.sessionStore != nil {
+		globals.sessionStore.EvictUser(uid, "")
+	}
+}
+
+func notifyUserStateChanged(tenantID types.TenantID, uid types.Uid, state types.ObjState) {
+	usersRemoveUser(tenantID, uid)
+	if globals.hub != nil && globals.hub.userStatus != nil {
+		globals.hub.userStatus <- &userStatusReq{tenantID: tenantID, forUser: uid, state: state}
+	}
+}
+
+func deleteUserByAdminCommand(tenantID types.TenantID, uid types.Uid, hard bool, _ time.Time) error {
+	evictUserSessions(uid)
+	usersRemoveUser(tenantID, uid)
+	if globals.hub != nil && globals.hub.unreg != nil {
+		done := make(chan bool)
+		globals.hub.unreg <- &topicUnreg{tenantID: tenantID, forUser: uid, del: hard, done: done}
+		<-done
+	}
+	return store.Users.Delete(tenantID, uid, hard)
+}
+
+func publishKafkaAdminCommandResult(ctx context.Context, command kafkaEnvelope, commandID, status, reason, message string, result map[string]any) error {
+	if commandID == "" {
+		commandID = command.EventID
+	}
+	payload := adminCommandResultPayload{
+		CommandID:      commandID,
+		CommandEventID: command.EventID,
+		Status:         status,
+		Reason:         reason,
+		Message:        message,
+		Result:         result,
+	}
+	encoded, err := json.Marshal(payload)
+	if err != nil {
+		return err
+	}
+	eventID := stableIMEventID("admin-command-result", command.EventID, status, reason)
+	return publishKafkaEnvelope(ctx, "admin_command_result", commandID, kafkaEnvelope{
+		EventID:        eventID,
+		TraceID:        command.TraceID,
+		TenantCode:     command.TenantCode,
+		SourceService:  "im-server",
+		TargetService:  command.SourceService,
+		EventType:      "admin.command." + status,
+		BizType:        command.BizType,
+		BizID:          command.BizID,
+		IdempotencyKey: command.IdempotencyKey,
+		OccurredAt:     time.Now().UTC(),
+		ParentEventID:  command.EventID,
+		Payload:        encoded,
+	})
+}
+
+func kafkaTenantID(tenantCode string) (types.TenantID, error) {
+	tenant, err := store.Tenants.GetByCode(strings.TrimSpace(tenantCode))
+	if err != nil {
+		return types.ZeroTenantID, err
+	}
+	if tenant == nil || tenant.ID.IsZero() {
+		return types.ZeroTenantID, types.ErrNotFound
+	}
+	return tenant.ID, nil
+}
+
+func markKafkaDeadLetter(ctx context.Context, tenantID types.TenantID, envelope kafkaEnvelope, consume *types.KafkaConsumeRecord, reason, message string) error {
+	encoded, err := json.Marshal(envelope)
+	if err != nil {
+		return err
+	}
+	retryCount := 0
+	sourceTopic := kafkaTopic("admin_command")
+	if consume != nil {
+		retryCount = consume.RetryCount
+		if strings.TrimSpace(consume.SourceTopic) != "" {
+			sourceTopic = consume.SourceTopic
+		}
+	}
+	deadID := stableIMEventID("dead", envelope.EventID, kafkaBus.config.ConsumerGroup, retryCount)
+	dead := &types.KafkaDeadLetter{
+		DeadID:        deadID,
+		EventID:       envelope.EventID,
+		ParentEventID: envelope.ParentEventID,
+		SourceTopic:   sourceTopic,
+		ConsumerGroup: kafkaBus.config.ConsumerGroup,
+		EventType:     envelope.EventType,
+		BizType:       envelope.BizType,
+		BizID:         envelope.BizID,
+		Envelope:      encoded,
+		RetryCount:    retryCount,
+		LastReason:    reason,
+		LastError:     message,
+		Status:        types.KafkaDeadStatusDead,
+	}
+	now := time.Now().UTC().Round(time.Millisecond)
+	if err = store.HTTPInfrastructure.MarkKafkaConsumeDead(tenantID, dead, now); err != nil {
+		return err
+	}
+	if auditErr := enqueueKafkaAuditEvent(ctx, tenantID, "system", types.ZeroUid, "kafka.dead_letter.created", "kafka_event", envelope.EventID, "dead", reason, map[string]any{
+		"dead_id":        deadID,
+		"event_type":     envelope.EventType,
+		"biz_type":       envelope.BizType,
+		"biz_id":         envelope.BizID,
+		"consumer_group": kafkaBus.config.ConsumerGroup,
+	}); auditErr != nil {
+		logs.Warn.Printf("kafka dead letter audit enqueue failed: event=%s dead=%s err=%v", envelope.EventID, deadID, auditErr)
+	}
+	deadPayload, err := json.Marshal(dead)
+	if err != nil {
+		return err
+	}
+	return publishKafkaEnvelope(ctx, "dead_letter", envelope.EventID, kafkaEnvelope{
+		EventID:       deadID,
+		TraceID:       envelope.TraceID,
+		TenantCode:    envelope.TenantCode,
+		SourceService: "im-server",
+		TargetService: "hotgo-admin",
+		EventType:     "kafka.dead_letter.created",
+		BizType:       envelope.BizType,
+		BizID:         envelope.BizID,
+		OccurredAt:    now,
+		ParentEventID: envelope.EventID,
+		Payload:       deadPayload,
+	})
+}
+
+func shouldDeadLetterKafkaCommand(reason string, retryCount int) bool {
+	if !kafkaCommandReasonRetryable(reason) {
+		return true
+	}
+	if kafkaBus == nil {
+		return true
+	}
+	return retryCount >= kafkaBus.config.MaxRetryCount
+}
+
+func kafkaCommandReasonRetryable(reason string) bool {
+	switch reason {
+	case "COMMAND_FAILED":
+		return true
+	default:
+		return false
+	}
+}
+
+func sleepKafkaRetryBackoff(retryCount int) {
+	if kafkaBus == nil || retryCount <= 0 {
+		return
+	}
+	backoffs := kafkaBus.config.RetryBackoffMS
+	index := retryCount - 1
+	if index >= len(backoffs) {
+		index = len(backoffs) - 1
+	}
+	delay := time.Duration(backoffs[index]) * time.Millisecond
+	if delay > 0 {
+		time.Sleep(delay)
+	}
+}
+
+func parseKafkaCommandUser(raw string) types.Uid {
+	raw = strings.TrimSpace(raw)
+	if raw == "" {
+		return types.ZeroUid
+	}
+	if id, err := strconv.ParseInt(raw, 10, 64); err == nil && id > 0 {
+		return store.EncodeUid(id)
+	}
+	uid := types.ParseUserId(raw)
+	if !uid.IsZero() {
+		return uid
+	}
+	return types.ParseUid(raw)
+}
+
+func kafkaDeadLetterResult(dead *types.KafkaDeadLetter) map[string]any {
+	if dead == nil {
+		return nil
+	}
+	return map[string]any{
+		"dead_id":      dead.DeadID,
+		"event_id":     dead.EventID,
+		"source_topic": dead.SourceTopic,
+		"event_type":   dead.EventType,
+		"biz_type":     dead.BizType,
+		"biz_id":       dead.BizID,
+		"status":       dead.Status,
+		"replay_count": dead.ReplayCount,
+		"updated_at":   dead.UpdatedAt,
+	}
+}
+
+func kafkaCommandErrorReason(err error) string {
+	if err == nil {
+		return ""
+	}
+	switch {
+	case errors.Is(err, types.ErrMalformed):
+		return "MALFORMED_COMMAND"
+	case errors.Is(err, types.ErrNotFound):
+		return "RESOURCE_NOT_FOUND"
+	case errors.Is(err, types.ErrPermissionDenied):
+		return "FORBIDDEN"
+	case errors.Is(err, types.ErrDuplicate):
+		return "COMMAND_DUPLICATED"
+	case errors.Is(err, types.ErrInvalidResponse):
+		return "GROUP_VERSION_CONFLICT"
+	default:
+		if strings.Contains(err.Error(), "unsupported admin command") {
+			return "UNSUPPORTED_COMMAND"
+		}
+		return "COMMAND_FAILED"
+	}
+}
+
+func firstNonEmpty(values ...string) string {
+	for _, value := range values {
+		if strings.TrimSpace(value) != "" {
+			return strings.TrimSpace(value)
+		}
+	}
+	return ""
+}

server/kafka_cache_invalidate.go

@@ -0,0 +1,130 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"strings"
+	"time"
+
+	"github.com/segmentio/kafka-go"
+	"github.com/tinode/chat/server/logs"
+	"github.com/tinode/chat/server/store"
+	"github.com/tinode/chat/server/store/types"
+)
+
+const (
+	cacheInvalidateEventAll                 = "cache.invalidate"
+	cacheInvalidateEventTenant              = "cache.tenant.invalidate"
+	cacheInvalidateEventRegistration        = "cache.registration.invalidate"
+	cacheInvalidateEventThirdParty          = "cache.third_party.invalidate"
+	cacheInvalidateEventDictionary          = "cache.dictionary.invalidate"
+	cacheInvalidateEventRedisObject         = "cache.redis_object.invalidate"
+	cacheInvalidateEventGroupHotMessages    = "cache.group_hot_messages.invalidate"
+	cacheInvalidateNamespaceTenant          = "tenant"
+	cacheInvalidateNamespaceRegistration    = "registration"
+	cacheInvalidateNamespaceThirdParty      = "third_party"
+	cacheInvalidateNamespaceDictionary      = "dictionary"
+	cacheInvalidateNamespaceGroupHotMessage = "group_hot_messages"
+)
+
+type cacheInvalidatePayload struct {
+	TenantCode string         `json:"tenant_code"`
+	TenantID   types.TenantID `json:"tenant_id"`
+	Namespace  string         `json:"namespace"`
+	Group      string         `json:"group"`
+	Key        string         `json:"key"`
+	Keys       []string       `json:"keys"`
+	Topic      string         `json:"topic"`
+	Ranges     []types.Range  `json:"ranges"`
+}
+
+func startKafkaCacheInvalidateConsumer() {
+	startKafkaConsumer("cache_invalidate", handleKafkaCacheInvalidateEvent)
+}
+
+func handleKafkaCacheInvalidateEvent(ctx context.Context, envelope kafkaEnvelope, _ kafka.Message) error {
+	if strings.TrimSpace(envelope.EventID) == "" {
+		return nil
+	}
+	tenantID, _ := kafkaTenantIDFromEnvelopeOrPayload(envelope)
+	if tenantID.IsZero() {
+		tenantID = types.TenantID(1)
+	}
+	consume, claimed, err := beginKafkaConsumeForTopic(tenantID, envelope, "cache_invalidate")
+	if err != nil {
+		return err
+	}
+	if !claimed {
+		logs.Info.Printf("kafka cache invalidate skipped: event=%s group=%s status=%s", envelope.EventID, kafkaBus.config.ConsumerGroup, consume.Status)
+		return nil
+	}
+	if err = executeKafkaCacheInvalidateEvent(envelope); err != nil {
+		reason := kafkaCommandErrorReason(err)
+		failed, markErr := store.HTTPInfrastructure.MarkKafkaConsumeFailed(tenantID, envelope.EventID, kafkaBus.config.ConsumerGroup, reason, err.Error(), time.Now().UTC().Round(time.Millisecond))
+		if markErr != nil {
+			return markErr
+		}
+		if failed.RetryCount >= kafkaBus.config.MaxRetryCount {
+			return markKafkaDeadLetter(ctx, tenantID, envelope, failed, reason, err.Error())
+		}
+		sleepKafkaRetryBackoff(failed.RetryCount)
+		return err
+	}
+	return store.HTTPInfrastructure.MarkKafkaConsumeSucceeded(tenantID, envelope.EventID, kafkaBus.config.ConsumerGroup, time.Now().UTC().Round(time.Millisecond))
+}
+
+func executeKafkaCacheInvalidateEvent(envelope kafkaEnvelope) error {
+	var payload cacheInvalidatePayload
+	if len(envelope.Payload) > 0 {
+		if err := json.Unmarshal(envelope.Payload, &payload); err != nil {
+			return err
+		}
+	}
+	tenantCode := firstNonEmpty(payload.TenantCode, envelope.TenantCode)
+	tenantID := payload.TenantID
+	if tenantID.IsZero() && tenantCode != "" {
+		if tenant, err := store.Tenants.GetByCode(tenantCode); err == nil && tenant != nil {
+			tenantID = tenant.ID
+		}
+	}
+
+	eventType := strings.TrimSpace(envelope.EventType)
+	namespace := strings.TrimSpace(payload.Namespace)
+	switch {
+	case eventType == cacheInvalidateEventTenant || namespace == cacheInvalidateNamespaceTenant:
+		store.InvalidateTenantConfigCache(tenantCode, tenantID)
+	case eventType == cacheInvalidateEventRegistration || namespace == cacheInvalidateNamespaceRegistration:
+		store.InvalidateRegistrationConfigCache()
+	case eventType == cacheInvalidateEventThirdParty || eventType == cacheInvalidateEventDictionary ||
+		namespace == cacheInvalidateNamespaceThirdParty || namespace == cacheInvalidateNamespaceDictionary:
+		store.InvalidateThirdPartyConfigCache(payload.Group)
+	case eventType == cacheInvalidateEventGroupHotMessages || namespace == cacheInvalidateNamespaceGroupHotMessage:
+		if tenantID.IsValid() && strings.TrimSpace(payload.Topic) != "" {
+			if err := store.InvalidateHotMessages(tenantID, strings.TrimSpace(payload.Topic), payload.Ranges); err != nil {
+				return err
+			}
+		}
+	case eventType == cacheInvalidateEventRedisObject || namespace != "":
+		if tenantID.IsValid() && namespace != "" {
+			keys := append([]string{}, payload.Keys...)
+			if strings.TrimSpace(payload.Key) != "" {
+				keys = append(keys, strings.TrimSpace(payload.Key))
+			}
+			for _, key := range keys {
+				key = strings.TrimSpace(key)
+				if key == "" {
+					continue
+				}
+				if err := store.DeleteCachedObject(tenantID, namespace, key); err != nil {
+					return err
+				}
+			}
+		}
+	default:
+		store.InvalidateTenantConfigCache(tenantCode, tenantID)
+		store.InvalidateThirdPartyConfigCache(payload.Group)
+		store.InvalidateRegistrationConfigCache()
+	}
+	logs.Info.Printf("kafka cache invalidate applied: event=%s type=%s tenant=%s namespace=%s group=%s", envelope.EventID, envelope.EventType, tenantCode, namespace, payload.Group)
+	return nil
+}

server/kafka_config_events.go

@@ -0,0 +1,122 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"strings"
+	"time"
+
+	"github.com/segmentio/kafka-go"
+	"github.com/tinode/chat/server/logs"
+	"github.com/tinode/chat/server/store"
+	"github.com/tinode/chat/server/store/types"
+)
+
+const (
+	configEventTenantUpdated         = "tenant.updated"
+	configEventTenantStatusChanged   = "tenant.status.changed"
+	configEventTenantLimitsUpdated   = "tenant.limits.updated"
+	configEventTenantFeaturesUpdated = "tenant.features.updated"
+	configEventDictionaryUpdated     = "dictionary.updated"
+	configEventThirdPartyUpdated     = "third_party.updated"
+	configEventRegistrationUpdated   = "registration.config.updated"
+)
+
+type configEventPayload struct {
+	TenantCode string         `json:"tenant_code"`
+	TenantID   types.TenantID `json:"tenant_id"`
+	Group      string         `json:"group"`
+	Keys       []string       `json:"keys"`
+}
+
+func startKafkaConfigEventConsumer() {
+	startKafkaConsumer("config_event", handleKafkaConfigEvent)
+}
+
+func handleKafkaConfigEvent(ctx context.Context, envelope kafkaEnvelope, _ kafka.Message) error {
+	if strings.TrimSpace(envelope.EventID) == "" {
+		return nil
+	}
+	tenantID, _ := kafkaTenantIDFromEnvelopeOrPayload(envelope)
+	if tenantID.IsZero() {
+		tenantID = types.TenantID(1)
+	}
+	consume, claimed, err := beginKafkaConsumeForTopic(tenantID, envelope, "config_event")
+	if err != nil {
+		return err
+	}
+	if !claimed {
+		logs.Info.Printf("kafka config event skipped: event=%s group=%s status=%s", envelope.EventID, kafkaBus.config.ConsumerGroup, consume.Status)
+		return nil
+	}
+	if err = executeKafkaConfigEvent(envelope); err != nil {
+		reason := kafkaCommandErrorReason(err)
+		failed, markErr := store.HTTPInfrastructure.MarkKafkaConsumeFailed(tenantID, envelope.EventID, kafkaBus.config.ConsumerGroup, reason, err.Error(), time.Now().UTC().Round(time.Millisecond))
+		if markErr != nil {
+			return markErr
+		}
+		if failed.RetryCount >= kafkaBus.config.MaxRetryCount {
+			return markKafkaDeadLetter(ctx, tenantID, envelope, failed, reason, err.Error())
+		}
+		sleepKafkaRetryBackoff(failed.RetryCount)
+		return err
+	}
+	return store.HTTPInfrastructure.MarkKafkaConsumeSucceeded(tenantID, envelope.EventID, kafkaBus.config.ConsumerGroup, time.Now().UTC().Round(time.Millisecond))
+}
+
+func executeKafkaConfigEvent(envelope kafkaEnvelope) error {
+	var payload configEventPayload
+	if len(envelope.Payload) > 0 {
+		if err := json.Unmarshal(envelope.Payload, &payload); err != nil {
+			return err
+		}
+	}
+	tenantCode := firstNonEmpty(payload.TenantCode, envelope.TenantCode)
+	tenantID := payload.TenantID
+	if tenantID.IsZero() && tenantCode != "" {
+		if tenant, err := store.Tenants.GetByCode(tenantCode); err == nil && tenant != nil {
+			tenantID = tenant.ID
+		}
+	}
+
+	switch strings.TrimSpace(envelope.EventType) {
+	case configEventTenantUpdated, configEventTenantStatusChanged, configEventTenantLimitsUpdated, configEventTenantFeaturesUpdated:
+		store.InvalidateTenantConfigCache(tenantCode, tenantID)
+	case configEventDictionaryUpdated, configEventThirdPartyUpdated:
+		store.InvalidateThirdPartyConfigCache(payload.Group)
+	case configEventRegistrationUpdated:
+		store.InvalidateRegistrationConfigCache()
+	default:
+		store.InvalidateTenantConfigCache(tenantCode, tenantID)
+		store.InvalidateThirdPartyConfigCache(payload.Group)
+		store.InvalidateRegistrationConfigCache()
+	}
+	logs.Info.Printf("kafka config event applied: event=%s type=%s tenant=%s group=%s", envelope.EventID, envelope.EventType, tenantCode, payload.Group)
+	return nil
+}
+
+func kafkaTenantIDFromEnvelopeOrPayload(envelope kafkaEnvelope) (types.TenantID, error) {
+	var payload configEventPayload
+	if len(envelope.Payload) > 0 {
+		_ = json.Unmarshal(envelope.Payload, &payload)
+	}
+	if payload.TenantID.IsValid() {
+		return payload.TenantID, nil
+	}
+	return kafkaTenantID(firstNonEmpty(envelope.TenantCode, payload.TenantCode))
+}
+
+func beginKafkaConsumeForTopic(tenantID types.TenantID, envelope kafkaEnvelope, topicName string) (*types.KafkaConsumeRecord, bool, error) {
+	if kafkaBus == nil {
+		return nil, false, types.ErrUnsupported
+	}
+	deadline := time.Now().UTC().Add(5 * time.Minute).Round(time.Millisecond)
+	return store.HTTPInfrastructure.BeginKafkaConsume(tenantID, &types.KafkaConsumeRecord{
+		EventID:       envelope.EventID,
+		ConsumerGroup: kafkaBus.config.ConsumerGroup,
+		SourceTopic:   kafkaTopic(topicName),
+		EventType:     envelope.EventType,
+		BizType:       envelope.BizType,
+		BizID:         envelope.BizID,
+	}, deadline)
+}

server/kafka_dead_letters.go

@@ -0,0 +1,41 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"strings"
+	"time"
+
+	"github.com/tinode/chat/server/store"
+	"github.com/tinode/chat/server/store/types"
+)
+
+func replayKafkaDeadLetter(ctx context.Context, tenantID types.TenantID, deadID string, actor types.Uid, note string) (*types.KafkaDeadLetter, error) {
+	now := time.Now().UTC().Round(time.Millisecond)
+	dead, err := store.HTTPInfrastructure.MarkKafkaDeadLetterReplay(tenantID, deadID, actor.UserId(), note, now)
+	if err != nil {
+		return nil, err
+	}
+	var envelope kafkaEnvelope
+	if err = json.Unmarshal(dead.Envelope, &envelope); err != nil {
+		return nil, err
+	}
+	if envelope.EventID == "" {
+		envelope.EventID = dead.EventID
+	}
+	if envelope.ParentEventID == "" {
+		envelope.ParentEventID = dead.ParentEventID
+	}
+	if err = publishKafkaEnvelope(ctx, strings.TrimSpace(dead.SourceTopic), envelope.EventID, envelope); err != nil {
+		return nil, err
+	}
+	if err = store.HTTPInfrastructure.MarkKafkaDeadLetter(tenantID, deadID, types.KafkaDeadStatusReplayed, actor.UserId(), note, now); err != nil {
+		return nil, err
+	}
+	return store.HTTPInfrastructure.GetKafkaDeadLetter(tenantID, deadID)
+}
+
+func resolveKafkaDeadLetter(tenantID types.TenantID, deadID, status string, actor types.Uid, note string) error {
+	now := time.Now().UTC().Round(time.Millisecond)
+	return store.HTTPInfrastructure.MarkKafkaDeadLetter(tenantID, deadID, status, actor.UserId(), note, now)
+}

server/kafka_file_lifecycle.go

@@ -0,0 +1,265 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/segmentio/kafka-go"
+	"github.com/tinode/chat/server/logs"
+	"github.com/tinode/chat/server/store"
+	"github.com/tinode/chat/server/store/types"
+)
+
+const (
+	kafkaFileLifecycleUploadCompleted = "file.upload.completed"
+	kafkaFileLifecycleAttachmentsBind = "file.attachments.bind"
+	kafkaFileLifecycleGroupFileCreate = "group.file.created"
+	kafkaFileLifecycleGroupFileDelete = "group.file.deleted"
+)
+
+type kafkaFileLifecyclePayload struct {
+	Action         string         `json:"action"`
+	FileID         string         `json:"file_id,omitempty"`
+	FileIDs        []string       `json:"file_ids,omitempty"`
+	AttachmentURLs []string       `json:"attachment_urls,omitempty"`
+	Topic          string         `json:"topic,omitempty"`
+	MessageID      string         `json:"message_id,omitempty"`
+	SeqID          int64          `json:"seq_id,omitempty"`
+	UserID         string         `json:"user_id,omitempty"`
+	GroupFileID    string         `json:"group_file_id,omitempty"`
+	FileType       string         `json:"file_type,omitempty"`
+	FileName       string         `json:"file_name,omitempty"`
+	MIMEType       string         `json:"mime_type,omitempty"`
+	FileSize       int64          `json:"file_size,omitempty"`
+	Storage        map[string]any `json:"storage,omitempty"`
+	OccurredAt     time.Time      `json:"occurred_at"`
+}
+
+func startKafkaFileLifecycleConsumer() {
+	startKafkaConsumer("file_lifecycle", handleKafkaFileLifecycle)
+}
+
+func enqueueKafkaFileUploaded(ctx context.Context, tenantID types.TenantID, fd *types.FileDef) error {
+	if !kafkaEnabled() || fd == nil || fd.TenantID != tenantID || fd.Status != types.UploadCompleted {
+		return nil
+	}
+	payload := kafkaFileLifecyclePayload{
+		Action:     kafkaFileLifecycleUploadCompleted,
+		FileID:     fd.Uid().String(),
+		UserID:     types.ParseUid(fd.User).UserId(),
+		MIMEType:   fd.MimeType,
+		FileSize:   fd.Size,
+		OccurredAt: fd.UpdatedAt,
+		Storage: map[string]any{
+			"location": fd.Location,
+			"etag":     fd.ETag,
+		},
+	}
+	return enqueueKafkaFileLifecycle(ctx, tenantID, kafkaFileLifecycleUploadCompleted, fd.Uid().String(), payload)
+}
+
+func enqueueKafkaFileAttachmentsBind(ctx context.Context, tenantID types.TenantID, msg *types.Message, attachmentURLs []string) error {
+	if !kafkaEnabled() || msg == nil || msg.TenantID != tenantID || len(attachmentURLs) == 0 {
+		return nil
+	}
+	payload := kafkaFileLifecyclePayload{
+		Action:         kafkaFileLifecycleAttachmentsBind,
+		AttachmentURLs: append([]string(nil), attachmentURLs...),
+		Topic:          msg.Topic,
+		MessageID:      msg.Uid().String(),
+		SeqID:          msg.SeqId,
+		UserID:         types.ParseUid(msg.From).UserId(),
+		FileType:       lifecycleFileTypeFromMessageHead(msg.Head),
+		OccurredAt:     msg.CreatedAt,
+	}
+	return enqueueKafkaFileLifecycle(ctx, tenantID, kafkaFileLifecycleAttachmentsBind, fmt.Sprintf("%s:%d", msg.Topic, msg.SeqId), payload)
+}
+
+func enqueueKafkaGroupFileLifecycle(ctx context.Context, tenantID types.TenantID, eventType string, file *types.GroupFile, actor types.Uid) error {
+	if !kafkaEnabled() || file == nil || file.Topic == "" || file.FileID == "" {
+		return nil
+	}
+	occurredAt := file.UpdatedAt
+	if occurredAt.IsZero() {
+		occurredAt = time.Now().UTC().Round(time.Millisecond)
+	}
+	payload := kafkaFileLifecyclePayload{
+		Action:      eventType,
+		FileID:      file.FileID,
+		FileIDs:     []string{file.FileID},
+		Topic:       file.Topic,
+		GroupFileID: file.GroupFileID,
+		SeqID:       file.SourceSeq,
+		UserID:      actor.UserId(),
+		FileType:    file.FileType,
+		FileName:    file.FileName,
+		MIMEType:    file.MIMEType,
+		FileSize:    file.FileSize,
+		OccurredAt:  occurredAt,
+	}
+	return enqueueKafkaFileLifecycle(ctx, tenantID, eventType, firstNonEmpty(file.GroupFileID, file.FileID), payload)
+}
+
+func enqueueKafkaFileLifecycle(_ context.Context, tenantID types.TenantID, eventType, bizID string, payload kafkaFileLifecyclePayload) error {
+	tenantCode := ""
+	if store.Tenants != nil {
+		if tenant, err := store.Tenants.GetByID(tenantID); err == nil && tenant != nil {
+			tenantCode = tenant.Code
+		} else {
+			logs.Warn.Printf("kafka file lifecycle: failed to resolve tenant code for tenant %d: %v", tenantID, err)
+		}
+	}
+	if payload.OccurredAt.IsZero() {
+		payload.OccurredAt = time.Now().UTC().Round(time.Millisecond)
+	}
+	encoded, err := json.Marshal(payload)
+	if err != nil {
+		return err
+	}
+	eventID := stableIMEventID("file-lifecycle", tenantID, eventType, bizID, payload.SeqID, strings.Join(payload.AttachmentURLs, ","))
+	err = enqueueKafkaOutbox(tenantID, "file_lifecycle", bizID, kafkaEnvelope{
+		EventID:       eventID,
+		TenantCode:    tenantCode,
+		SourceService: "im-server",
+		TargetService: "im-server",
+		EventType:     eventType,
+		BizType:       "file",
+		BizID:         bizID,
+		OccurredAt:    payload.OccurredAt,
+		Payload:       encoded,
+	})
+	if err != nil {
+		return err
+	}
+	actor := types.ParseUserId(payload.UserID)
+	if auditErr := enqueueKafkaAuditEvent(context.Background(), tenantID, "user", actor, eventType, "file", bizID, "succeeded", "", map[string]any{
+		"file_id":       payload.FileID,
+		"file_ids":      payload.FileIDs,
+		"topic":         payload.Topic,
+		"group_file_id": payload.GroupFileID,
+		"seq_id":        payload.SeqID,
+	}); auditErr != nil {
+		logs.Warn.Printf("kafka file lifecycle: failed to enqueue audit event type=%s biz=%s: %v", eventType, bizID, auditErr)
+	}
+	return nil
+}
+
+func handleKafkaFileLifecycle(ctx context.Context, envelope kafkaEnvelope, _ kafka.Message) error {
+	tenantID, err := kafkaTenantID(envelope.TenantCode)
+	if err != nil {
+		return err
+	}
+	consume, claimed, err := beginKafkaConsumeForTopic(tenantID, envelope, "file_lifecycle")
+	if err != nil || !claimed {
+		return err
+	}
+	var payload kafkaFileLifecyclePayload
+	if err = json.Unmarshal(envelope.Payload, &payload); err != nil {
+		return markKafkaDeadLetter(ctx, tenantID, envelope, consume, "MALFORMED_PAYLOAD", err.Error())
+	}
+	if err = executeKafkaFileLifecycle(tenantID, payload); err != nil {
+		failed, markErr := store.HTTPInfrastructure.MarkKafkaConsumeFailed(tenantID, envelope.EventID, kafkaBus.config.ConsumerGroup,
+			kafkaCommandErrorReason(err), err.Error(), time.Now().UTC().Round(time.Millisecond))
+		if markErr != nil {
+			return markErr
+		}
+		if failed != nil && failed.RetryCount >= kafkaBus.config.MaxRetryCount {
+			return markKafkaDeadLetter(ctx, tenantID, envelope, failed, kafkaCommandErrorReason(err), err.Error())
+		}
+		sleepKafkaRetryBackoff(failed.RetryCount)
+		return err
+	}
+	return store.HTTPInfrastructure.MarkKafkaConsumeSucceeded(tenantID, envelope.EventID, kafkaBus.config.ConsumerGroup, time.Now().UTC().Round(time.Millisecond))
+}
+
+func executeKafkaFileLifecycle(tenantID types.TenantID, payload kafkaFileLifecyclePayload) error {
+	switch payload.Action {
+	case kafkaFileLifecycleAttachmentsBind:
+		return executeKafkaFileAttachmentsBind(tenantID, payload)
+	case kafkaFileLifecycleUploadCompleted, kafkaFileLifecycleGroupFileCreate, kafkaFileLifecycleGroupFileDelete:
+		return nil
+	default:
+		return types.ErrMalformed
+	}
+}
+
+func executeKafkaFileAttachmentsBind(tenantID types.TenantID, payload kafkaFileLifecyclePayload) error {
+	if payload.Topic == "" || payload.MessageID == "" || len(payload.AttachmentURLs) == 0 {
+		return types.ErrMalformed
+	}
+	msgID := types.ParseUid(payload.MessageID)
+	if msgID.IsZero() {
+		return types.ErrMalformed
+	}
+	if types.GetTopicCat(payload.Topic) != types.TopicCatGrp {
+		return store.Files.LinkAttachments(tenantID, "", msgID, payload.AttachmentURLs)
+	}
+	uploader := types.ParseUserId(payload.UserID)
+	if uploader.IsZero() {
+		return types.ErrMalformed
+	}
+	for _, url := range payload.AttachmentURLs {
+		fid := store.Store.GetMediaHandler().GetIdFromUrl(url)
+		if fid.IsZero() {
+			continue
+		}
+		_, err := store.GroupBusiness.CreateFile(tenantID, &types.GroupFile{
+			Topic:          payload.Topic,
+			FileID:         fid.String(),
+			SourceTopic:    payload.Topic,
+			SourceSeq:      payload.SeqID,
+			UploaderID:     uploader,
+			FileType:       lifecycleFileType(payload.FileType),
+			IdempotencyKey: fmt.Sprintf("%s:%d:%s", payload.Topic, payload.SeqID, fid.String()),
+		})
+		if err != nil && !errors.Is(err, types.ErrDuplicate) {
+			return err
+		}
+	}
+	return nil
+}
+
+func lifecycleFileTypeFromMessageHead(head types.KVMap) string {
+	msgType := lifecycleHeaderString(head, "x-im-type")
+	if msgType == "" {
+		msgType = lifecycleHeaderString(head, "type")
+	}
+	if msgType == "" {
+		return "file"
+	}
+	return lifecycleFileType(msgType)
+}
+
+func lifecycleFileType(raw string) string {
+	switch lifecycleCanonicalMessageType(raw) {
+	case "image", "audio", "voice", "video":
+		return lifecycleCanonicalMessageType(raw)
+	default:
+		return "file"
+	}
+}
+
+func lifecycleHeaderString(head types.KVMap, key string) string {
+	if head == nil {
+		return ""
+	}
+	value, ok := head[key]
+	if !ok {
+		return ""
+	}
+	str, _ := value.(string)
+	return strings.TrimSpace(str)
+}
+
+func lifecycleCanonicalMessageType(raw string) string {
+	switch strings.ToLower(strings.TrimSpace(raw)) {
+	case "audio":
+		return "voice"
+	default:
+		return strings.ToLower(strings.TrimSpace(raw))
+	}
+}

server/kafka_message_persist.go

@@ -0,0 +1,153 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"time"
+
+	"github.com/segmentio/kafka-go"
+	"github.com/tinode/chat/server/logs"
+	"github.com/tinode/chat/server/store"
+	"github.com/tinode/chat/server/store/types"
+)
+
+type kafkaMessagePersistPayload struct {
+	Message        types.Message `json:"message"`
+	AttachmentURLs []string      `json:"attachment_urls,omitempty"`
+	ReadBySender   bool          `json:"read_by_sender,omitempty"`
+	MessageKind    string        `json:"message_kind"`
+}
+
+func startKafkaMessagePersistConsumer() {
+	startKafkaConsumer("message_persist", handleKafkaMessagePersist)
+}
+
+func saveMessageMaybeKafka(ctx context.Context, tenantID types.TenantID, msg *types.Message, attachmentURLs []string, readBySender bool) (*types.MessageSaveResult, error) {
+	if !kafkaEnabled() {
+		return store.Messages.Save(tenantID, msg, attachmentURLs, readBySender)
+	}
+	saveResult, err := store.PrepareMessagePersist(tenantID, msg, attachmentURLs, readBySender)
+	if err != nil || saveResult == nil || saveResult.Duplicate {
+		return saveResult, err
+	}
+	if readBySender && !types.ParseUid(msg.From).IsZero() {
+		saveResult.MarkedReadBySender = true
+	}
+	if err = enqueueKafkaMessagePersist(ctx, tenantID, msg, attachmentURLs, readBySender); err != nil {
+		return nil, err
+	}
+	return saveResult, nil
+}
+
+func enqueueKafkaMessagePersist(_ context.Context, tenantID types.TenantID, msg *types.Message, attachmentURLs []string, readBySender bool) error {
+	tenantCode := ""
+	if store.Tenants != nil {
+		if tenant, err := store.Tenants.GetByID(tenantID); err == nil && tenant != nil {
+			tenantCode = tenant.Code
+		} else {
+			logs.Warn.Printf("kafka message persist: failed to resolve tenant code for tenant %d: %v", tenantID, err)
+		}
+	}
+	kind := "p2p"
+	if types.GetTopicCat(msg.Topic) == types.TopicCatGrp {
+		kind = "group"
+	}
+	payload, err := json.Marshal(kafkaMessagePersistPayload{
+		Message:        *msg,
+		AttachmentURLs: attachmentURLs,
+		ReadBySender:   readBySender,
+		MessageKind:    kind,
+	})
+	if err != nil {
+		return err
+	}
+	eventID := fmt.Sprintf("msg.persist.%d.%s.%d", tenantID, msg.Topic, msg.SeqId)
+	envelope := kafkaEnvelope{
+		EventID:        eventID,
+		TenantCode:     tenantCode,
+		SourceService:  "im-server",
+		TargetService:  "im-server",
+		EventType:      "message.persist." + kind,
+		BizType:        "message",
+		BizID:          fmt.Sprintf("%s:%d", msg.Topic, msg.SeqId),
+		IdempotencyKey: msg.ClientMsgID,
+		Version:        msg.SeqId,
+		OccurredAt:     msg.CreatedAt,
+		Actor:          &kafkaActor{Type: "user", ID: types.ParseUid(msg.From).UserId()},
+		Payload:        payload,
+	}
+	return enqueueKafkaOutbox(tenantID, "message_persist", msg.Topic, envelope)
+}
+
+func handleKafkaMessagePersist(ctx context.Context, envelope kafkaEnvelope, _ kafka.Message) error {
+	tenantID, err := kafkaTenantID(envelope.TenantCode)
+	if err != nil {
+		return err
+	}
+	consume, claimed, err := beginKafkaConsumeForTopic(tenantID, envelope, "message_persist")
+	if err != nil || !claimed {
+		return err
+	}
+	var payload kafkaMessagePersistPayload
+	if err = json.Unmarshal(envelope.Payload, &payload); err != nil {
+		return markKafkaDeadLetter(ctx, tenantID, envelope, consume, "MALFORMED_PAYLOAD", err.Error())
+	}
+	if payload.Message.TenantID == types.ZeroTenantID {
+		payload.Message.TenantID = tenantID
+	}
+	if payload.Message.TenantID != tenantID {
+		return markKafkaDeadLetter(ctx, tenantID, envelope, consume, "TENANT_MISMATCH", "message tenant does not match envelope tenant")
+	}
+	if err = validateKafkaMessageKind(payload); err != nil {
+		return markKafkaDeadLetter(ctx, tenantID, envelope, consume, "MALFORMED_PAYLOAD", err.Error())
+	}
+	if _, err = store.PersistPreparedMessage(tenantID, &payload.Message, nil, payload.ReadBySender); err != nil && !errors.Is(err, types.ErrDuplicate) {
+		failed, markErr := store.HTTPInfrastructure.MarkKafkaConsumeFailed(tenantID, envelope.EventID, kafkaBus.config.ConsumerGroup,
+			kafkaCommandErrorReason(err), err.Error(), time.Now().UTC().Round(time.Millisecond))
+		if markErr != nil {
+			return markErr
+		}
+		if failed != nil && failed.RetryCount >= kafkaBus.config.MaxRetryCount {
+			return markKafkaDeadLetter(ctx, tenantID, envelope, failed, kafkaCommandErrorReason(err), err.Error())
+		}
+		sleepKafkaRetryBackoff(failed.RetryCount)
+		return err
+	}
+	if len(payload.AttachmentURLs) > 0 {
+		if err = enqueueKafkaFileAttachmentsBind(ctx, tenantID, &payload.Message, payload.AttachmentURLs); err != nil {
+			failed, markErr := store.HTTPInfrastructure.MarkKafkaConsumeFailed(tenantID, envelope.EventID, kafkaBus.config.ConsumerGroup,
+				"KAFKA_FILE_LIFECYCLE_FAILED", err.Error(), time.Now().UTC().Round(time.Millisecond))
+			if markErr != nil {
+				return markErr
+			}
+			if failed != nil && failed.RetryCount >= kafkaBus.config.MaxRetryCount {
+				return markKafkaDeadLetter(ctx, tenantID, envelope, failed, "KAFKA_FILE_LIFECYCLE_FAILED", err.Error())
+			}
+			sleepKafkaRetryBackoff(failed.RetryCount)
+			return err
+		}
+	}
+	return store.HTTPInfrastructure.MarkKafkaConsumeSucceeded(tenantID, envelope.EventID, kafkaBus.config.ConsumerGroup, time.Now().UTC().Round(time.Millisecond))
+}
+
+func validateKafkaMessageKind(payload kafkaMessagePersistPayload) error {
+	if payload.Message.Topic == "" || payload.Message.SeqId <= 0 {
+		return types.ErrMalformed
+	}
+	isGroup := types.GetTopicCat(payload.Message.Topic) == types.TopicCatGrp
+	switch payload.MessageKind {
+	case "group":
+		if !isGroup {
+			return types.ErrMalformed
+		}
+	case "p2p", "":
+		if isGroup {
+			return types.ErrMalformed
+		}
+	default:
+		return types.ErrMalformed
+	}
+	return nil
+}

server/kafka_outbox.go

@@ -0,0 +1,126 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"time"
+
+	"github.com/segmentio/kafka-go"
+	"github.com/tinode/chat/server/logs"
+	"github.com/tinode/chat/server/store"
+	"github.com/tinode/chat/server/store/types"
+)
+
+const (
+	kafkaOutboxPollInterval = 2 * time.Second
+	kafkaOutboxTenantLimit  = 1000
+	kafkaOutboxBatchLimit   = 100
+)
+
+func startKafkaOutboxPoller() {
+	if !kafkaEnabled() {
+		return
+	}
+	kafkaBus.wg.Add(1)
+	go func() {
+		defer kafkaBus.wg.Done()
+		ticker := time.NewTicker(kafkaOutboxPollInterval)
+		defer ticker.Stop()
+		for {
+			select {
+			case <-kafkaBus.ctx.Done():
+				return
+			default:
+				drainKafkaOutbox(kafkaBus.ctx)
+			}
+			select {
+			case <-kafkaBus.ctx.Done():
+				return
+			case <-ticker.C:
+			}
+		}
+	}()
+}
+
+func enqueueKafkaOutbox(tenantID types.TenantID, topicName, key string, envelope kafkaEnvelope) error {
+	if envelope.OccurredAt.IsZero() {
+		envelope.OccurredAt = time.Now().UTC()
+	}
+	raw, err := json.Marshal(envelope)
+	if err != nil {
+		return err
+	}
+	return store.HTTPInfrastructure.CreateKafkaOutbox(tenantID, &types.KafkaOutboxRecord{
+		EventID:     envelope.EventID,
+		Topic:       topicName,
+		EventType:   envelope.EventType,
+		BizType:     envelope.BizType,
+		BizID:       envelope.BizID,
+		Key:         key,
+		Envelope:    raw,
+		Status:      types.KafkaOutboxPending,
+		NextRetryAt: nil,
+	})
+}
+
+func drainKafkaOutbox(ctx context.Context) {
+	if store.Tenants == nil || store.HTTPInfrastructure == nil {
+		return
+	}
+	tenants, err := store.Tenants.ListActive(kafkaOutboxTenantLimit)
+	if err != nil {
+		logs.Warn.Printf("kafka outbox: list active tenants: %v", err)
+		return
+	}
+	now := time.Now().UTC().Round(time.Millisecond)
+	for _, tenant := range tenants {
+		if tenant.ID.IsZero() {
+			continue
+		}
+		records, err := store.HTTPInfrastructure.ListPendingKafkaOutbox(tenant.ID, kafkaOutboxBatchLimit, now)
+		if err != nil {
+			logs.Warn.Printf("kafka outbox: list tenant=%d: %v", tenant.ID, err)
+			continue
+		}
+		for _, record := range records {
+			if err = publishKafkaOutboxRecord(ctx, record); err != nil {
+				nextRetryAt := now.Add(kafkaOutboxRetryDelay(record.Attempts + 1))
+				if markErr := store.HTTPInfrastructure.MarkKafkaOutboxFailed(tenant.ID, record.EventID,
+					"KAFKA_PUBLISH_FAILED", err.Error(), nextRetryAt, now); markErr != nil {
+					logs.Warn.Printf("kafka outbox: mark failed event=%s tenant=%d: %v", record.EventID, tenant.ID, markErr)
+				}
+				continue
+			}
+			if err = store.HTTPInfrastructure.MarkKafkaOutboxSent(tenant.ID, record.EventID, time.Now().UTC().Round(time.Millisecond)); err != nil {
+				logs.Warn.Printf("kafka outbox: mark sent event=%s tenant=%d: %v", record.EventID, tenant.ID, err)
+			}
+		}
+	}
+}
+
+func publishKafkaOutboxRecord(ctx context.Context, record types.KafkaOutboxRecord) error {
+	topic := kafkaTopic(record.Topic)
+	if topic == "" {
+		topic = record.Topic
+	}
+	return kafkaBus.writer.WriteMessages(ctx, kafka.Message{
+		Topic: topic,
+		Key:   []byte(record.Key),
+		Value: record.Envelope,
+		Time:  time.Now().UTC(),
+	})
+}
+
+func kafkaOutboxRetryDelay(attempt int) time.Duration {
+	if attempt <= 0 {
+		attempt = 1
+	}
+	if kafkaBus == nil || len(kafkaBus.config.RetryBackoffMS) == 0 {
+		return time.Duration(attempt) * 10 * time.Second
+	}
+	index := attempt - 1
+	if index >= len(kafkaBus.config.RetryBackoffMS) {
+		index = len(kafkaBus.config.RetryBackoffMS) - 1
+	}
+	return time.Duration(kafkaBus.config.RetryBackoffMS[index]) * time.Millisecond
+}

server/kafka_runtime.go

@@ -0,0 +1,308 @@
+package main
+
+import (
+	"context"
+	"crypto/tls"
+	"encoding/json"
+	"errors"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/segmentio/kafka-go"
+	"github.com/tinode/chat/server/logs"
+)
+
+const (
+	kafkaTopicAdminCommand       = "im.admin.command"
+	kafkaTopicAdminCommandResult = "im.admin.command.result"
+	kafkaTopicConfigEvent        = "im.config.event"
+	kafkaTopicBusinessEvent      = "im.business.event"
+	kafkaTopicMessagePersist     = "im.message.persist"
+	kafkaTopicMessagePush        = "im.message.push"
+	kafkaTopicFileSecurity       = "im.file.security"
+	kafkaTopicFileLifecycle      = "im.file.lifecycle"
+	kafkaTopicWorkbenchEvent     = "im.workbench.event"
+	kafkaTopicCacheInvalidate    = "im.cache.invalidate"
+	kafkaTopicAuditEvent         = "im.audit.event"
+	kafkaTopicDeadLetter         = "im.dead.letter"
+)
+
+type kafkaConfig struct {
+	Enabled          bool              `json:"enabled"`
+	Brokers          []string          `json:"brokers"`
+	ClientID         string            `json:"client_id"`
+	ConsumerGroup    string            `json:"consumer_group"`
+	Topics           map[string]string `json:"topics"`
+	DialTimeoutMS    int               `json:"dial_timeout_ms"`
+	WriteTimeoutMS   int               `json:"write_timeout_ms"`
+	ReadTimeoutMS    int               `json:"read_timeout_ms"`
+	RequiredAcks     int               `json:"required_acks"`
+	AllowAutoTopic   bool              `json:"allow_auto_topic_creation"`
+	TLSEnabled       bool              `json:"tls_enabled"`
+	MaxRetryCount    int               `json:"max_retry_count"`
+	RetryBackoffMS   []int             `json:"retry_backoff_ms"`
+	DeadRetentionSec int               `json:"dead_letter_retention_seconds"`
+}
+
+type kafkaEnvelope struct {
+	EventID        string          `json:"event_id"`
+	TraceID        string          `json:"trace_id,omitempty"`
+	TenantCode     string          `json:"tenant_code,omitempty"`
+	SourceService  string          `json:"source_service,omitempty"`
+	TargetService  string          `json:"target_service,omitempty"`
+	EventType      string          `json:"event_type"`
+	BizType        string          `json:"biz_type,omitempty"`
+	BizID          string          `json:"biz_id,omitempty"`
+	IdempotencyKey string          `json:"idempotency_key,omitempty"`
+	Version        int64           `json:"version,omitempty"`
+	OccurredAt     time.Time       `json:"occurred_at"`
+	Actor          *kafkaActor     `json:"actor,omitempty"`
+	Payload        json.RawMessage `json:"payload,omitempty"`
+	ParentEventID  string          `json:"parent_event_id,omitempty"`
+}
+
+type kafkaActor struct {
+	Type        string `json:"type,omitempty"`
+	ID          string `json:"id,omitempty"`
+	DisplayName string `json:"display_name,omitempty"`
+}
+
+type kafkaRuntime struct {
+	config kafkaConfig
+	topics map[string]string
+
+	writer *kafka.Writer
+	ctx    context.Context
+	cancel context.CancelFunc
+	wg     sync.WaitGroup
+}
+
+var kafkaBus *kafkaRuntime
+
+func initKafkaRuntime(raw json.RawMessage) error {
+	if len(raw) == 0 {
+		kafkaBus = nil
+		logs.Info.Println("Kafka disabled: no config")
+		return nil
+	}
+	var cfg kafkaConfig
+	if err := json.Unmarshal(raw, &cfg); err != nil {
+		return err
+	}
+	if !cfg.Enabled {
+		kafkaBus = nil
+		logs.Info.Println("Kafka disabled")
+		return nil
+	}
+	normalizeKafkaConfig(&cfg)
+	if len(cfg.Brokers) == 0 {
+		return errors.New("kafka: enabled but brokers are not configured")
+	}
+
+	ctx, cancel := context.WithCancel(context.Background())
+	rt := &kafkaRuntime{
+		config: cfg,
+		topics: kafkaTopics(cfg.Topics),
+		ctx:    ctx,
+		cancel: cancel,
+	}
+	transport := &kafka.Transport{
+		ClientID: cfg.ClientID,
+	}
+	if cfg.TLSEnabled {
+		transport.TLS = &tls.Config{MinVersion: tls.VersionTLS12}
+	}
+	rt.writer = &kafka.Writer{
+		Addr:                   kafka.TCP(cfg.Brokers...),
+		Balancer:               &kafka.Hash{},
+		Transport:              transport,
+		RequiredAcks:           kafka.RequiredAcks(cfg.RequiredAcks),
+		AllowAutoTopicCreation: cfg.AllowAutoTopic,
+		WriteTimeout:           time.Duration(cfg.WriteTimeoutMS) * time.Millisecond,
+		ReadTimeout:            time.Duration(cfg.ReadTimeoutMS) * time.Millisecond,
+	}
+
+	if err := rt.ping(context.Background()); err != nil {
+		cancel()
+		_ = rt.writer.Close()
+		return err
+	}
+	kafkaBus = rt
+	logs.Info.Printf("Kafka enabled: brokers=%s client_id=%s group=%s", strings.Join(cfg.Brokers, ","), cfg.ClientID, cfg.ConsumerGroup)
+	return nil
+}
+
+func normalizeKafkaConfig(cfg *kafkaConfig) {
+	if cfg.ClientID == "" {
+		cfg.ClientID = "im-server"
+	}
+	if cfg.ConsumerGroup == "" {
+		cfg.ConsumerGroup = "im-server"
+	}
+	if cfg.DialTimeoutMS <= 0 {
+		cfg.DialTimeoutMS = 3000
+	}
+	if cfg.WriteTimeoutMS <= 0 {
+		cfg.WriteTimeoutMS = 3000
+	}
+	if cfg.ReadTimeoutMS <= 0 {
+		cfg.ReadTimeoutMS = 3000
+	}
+	if cfg.RequiredAcks == 0 {
+		cfg.RequiredAcks = int(kafka.RequireOne)
+	}
+	if cfg.MaxRetryCount <= 0 {
+		cfg.MaxRetryCount = 5
+	}
+	if len(cfg.RetryBackoffMS) == 0 {
+		cfg.RetryBackoffMS = []int{10000, 30000, 60000, 300000, 900000}
+	}
+	if cfg.DeadRetentionSec <= 0 {
+		cfg.DeadRetentionSec = 90 * 24 * 3600
+	}
+}
+
+func kafkaTopics(overrides map[string]string) map[string]string {
+	topics := map[string]string{
+		"admin_command":        kafkaTopicAdminCommand,
+		"admin_command_result": kafkaTopicAdminCommandResult,
+		"config_event":         kafkaTopicConfigEvent,
+		"business_event":       kafkaTopicBusinessEvent,
+		"message_persist":      kafkaTopicMessagePersist,
+		"message_push":         kafkaTopicMessagePush,
+		"file_security":        kafkaTopicFileSecurity,
+		"file_lifecycle":       kafkaTopicFileLifecycle,
+		"workbench_event":      kafkaTopicWorkbenchEvent,
+		"cache_invalidate":     kafkaTopicCacheInvalidate,
+		"audit_event":          kafkaTopicAuditEvent,
+		"dead_letter":          kafkaTopicDeadLetter,
+	}
+	for key, value := range overrides {
+		if value != "" {
+			topics[key] = value
+		}
+	}
+	return topics
+}
+
+func (rt *kafkaRuntime) ping(ctx context.Context) error {
+	timeout := time.Duration(rt.config.DialTimeoutMS) * time.Millisecond
+	dialer := rt.dialer()
+	pingCtx, cancel := context.WithTimeout(ctx, timeout)
+	defer cancel()
+	conn, err := dialer.DialContext(pingCtx, "tcp", rt.config.Brokers[0])
+	if err != nil {
+		return err
+	}
+	return conn.Close()
+}
+
+func kafkaEnabled() bool {
+	return kafkaBus != nil
+}
+
+func kafkaTopic(name string) string {
+	if kafkaBus == nil {
+		return ""
+	}
+	return kafkaBus.topics[name]
+}
+
+func publishKafkaEnvelope(ctx context.Context, topicName string, key string, envelope kafkaEnvelope) error {
+	if kafkaBus == nil {
+		return nil
+	}
+	if envelope.OccurredAt.IsZero() {
+		envelope.OccurredAt = time.Now().UTC()
+	}
+	topic := kafkaBus.topics[topicName]
+	if topic == "" {
+		topic = topicName
+	}
+	value, err := json.Marshal(envelope)
+	if err != nil {
+		return err
+	}
+	return kafkaBus.writer.WriteMessages(ctx, kafka.Message{
+		Topic: topic,
+		Key:   []byte(key),
+		Value: value,
+		Time:  envelope.OccurredAt,
+	})
+}
+
+type kafkaMessageHandler func(context.Context, kafkaEnvelope, kafka.Message) error
+
+func startKafkaConsumer(topicName string, handler kafkaMessageHandler) {
+	if kafkaBus == nil || handler == nil {
+		return
+	}
+	topic := kafkaBus.topics[topicName]
+	if topic == "" {
+		topic = topicName
+	}
+	reader := kafka.NewReader(kafka.ReaderConfig{
+		Brokers:     kafkaBus.config.Brokers,
+		GroupID:     kafkaBus.config.ConsumerGroup,
+		Topic:       topic,
+		StartOffset: kafka.FirstOffset,
+		MinBytes:    1,
+		MaxBytes:    10e6,
+		MaxWait:     time.Second,
+		Dialer:      kafkaBus.dialer(),
+	})
+	kafkaBus.wg.Add(1)
+	go func() {
+		defer kafkaBus.wg.Done()
+		defer reader.Close()
+		for {
+			msg, err := reader.FetchMessage(kafkaBus.ctx)
+			if err != nil {
+				if errors.Is(err, context.Canceled) || errors.Is(kafkaBus.ctx.Err(), context.Canceled) {
+					return
+				}
+				logs.Err.Println("Kafka fetch failed:", err)
+				continue
+			}
+			var envelope kafkaEnvelope
+			if err = json.Unmarshal(msg.Value, &envelope); err != nil {
+				logs.Err.Println("Kafka envelope decode failed:", err)
+				_ = reader.CommitMessages(context.Background(), msg)
+				continue
+			}
+			if err = handler(kafkaBus.ctx, envelope, msg); err != nil {
+				logs.Err.Println("Kafka handler failed:", envelope.EventID, envelope.EventType, err)
+				continue
+			}
+			if err = reader.CommitMessages(context.Background(), msg); err != nil {
+				logs.Err.Println("Kafka commit failed:", err)
+			}
+		}
+	}()
+	logs.Info.Printf("Kafka consumer started: topic=%s group=%s", topic, kafkaBus.config.ConsumerGroup)
+}
+
+func (rt *kafkaRuntime) dialer() *kafka.Dialer {
+	dialer := &kafka.Dialer{
+		ClientID: rt.config.ClientID,
+		Timeout:  time.Duration(rt.config.DialTimeoutMS) * time.Millisecond,
+	}
+	if rt.config.TLSEnabled {
+		dialer.TLS = &tls.Config{MinVersion: tls.VersionTLS12}
+	}
+	return dialer
+}
+
+func stopKafkaRuntime() {
+	if kafkaBus == nil {
+		return
+	}
+	kafkaBus.cancel()
+	kafkaBus.wg.Wait()
+	if kafkaBus.writer != nil {
+		_ = kafkaBus.writer.Close()
+	}
+	logs.Info.Println("Stopped Kafka runtime")
+	kafkaBus = nil
+}

server/kafka_workbench_audit.go

@@ -0,0 +1,171 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"time"
+
+	"github.com/segmentio/kafka-go"
+	"github.com/tinode/chat/server/logs"
+	"github.com/tinode/chat/server/store"
+	"github.com/tinode/chat/server/store/types"
+)
+
+type kafkaWorkbenchPayload struct {
+	Module        string         `json:"module"`
+	Action        string         `json:"action"`
+	ResourceID    string         `json:"resource_id"`
+	ActorUserID   string         `json:"actor_user_id"`
+	TargetUserIDs []string       `json:"target_user_ids,omitempty"`
+	Title         string         `json:"title,omitempty"`
+	Summary       string         `json:"summary,omitempty"`
+	Data          map[string]any `json:"data,omitempty"`
+	OccurredAt    time.Time      `json:"occurred_at"`
+}
+
+type kafkaAuditPayload struct {
+	ActorType  string         `json:"actor_type"`
+	ActorID    string         `json:"actor_id"`
+	Action     string         `json:"action"`
+	Resource   string         `json:"resource"`
+	ResourceID string         `json:"resource_id"`
+	Result     string         `json:"result"`
+	Reason     string         `json:"reason,omitempty"`
+	Data       map[string]any `json:"data,omitempty"`
+	OccurredAt time.Time      `json:"occurred_at"`
+}
+
+func startKafkaWorkbenchEventConsumer() {
+	startKafkaConsumer("workbench_event", handleKafkaWorkbenchEvent)
+}
+
+func enqueueKafkaWorkbenchEvent(ctx context.Context, tenantID types.TenantID, actor types.Uid, module, action, resourceID string,
+	targets []types.Uid, title, summary string, data map[string]any) error {
+	if !kafkaEnabled() {
+		return nil
+	}
+	targetUserIDs := make([]string, 0, len(targets))
+	for _, target := range targets {
+		if !target.IsZero() {
+			targetUserIDs = append(targetUserIDs, target.UserId())
+		}
+	}
+	payload := kafkaWorkbenchPayload{
+		Module:        module,
+		Action:        action,
+		ResourceID:    resourceID,
+		ActorUserID:   actor.UserId(),
+		TargetUserIDs: targetUserIDs,
+		Title:         title,
+		Summary:       summary,
+		Data:          data,
+		OccurredAt:    time.Now().UTC().Round(time.Millisecond),
+	}
+	return enqueueKafkaTypedEvent(ctx, tenantID, "workbench_event", "workbench."+action, "workbench", resourceID, actor, payload)
+}
+
+func enqueueKafkaAuditEvent(ctx context.Context, tenantID types.TenantID, actorType string, actor types.Uid,
+	action, resource, resourceID, result, reason string, data map[string]any) error {
+	if !kafkaEnabled() {
+		return nil
+	}
+	actorID := ""
+	if !actor.IsZero() {
+		actorID = actor.UserId()
+	}
+	payload := kafkaAuditPayload{
+		ActorType:  actorType,
+		ActorID:    actorID,
+		Action:     action,
+		Resource:   resource,
+		ResourceID: resourceID,
+		Result:     result,
+		Reason:     reason,
+		Data:       data,
+		OccurredAt: time.Now().UTC().Round(time.Millisecond),
+	}
+	return enqueueKafkaTypedEvent(ctx, tenantID, "audit_event", "audit."+action, "audit", resource+":"+resourceID, actor, payload)
+}
+
+func enqueueKafkaTypedEvent(_ context.Context, tenantID types.TenantID, topicName, eventType, bizType, bizID string, actor types.Uid, payload any) error {
+	tenantCode := ""
+	if store.Tenants != nil {
+		if tenant, err := store.Tenants.GetByID(tenantID); err == nil && tenant != nil {
+			tenantCode = tenant.Code
+		} else {
+			logs.Warn.Printf("kafka event: failed to resolve tenant code for tenant %d: %v", tenantID, err)
+		}
+	}
+	encoded, err := json.Marshal(payload)
+	if err != nil {
+		return err
+	}
+	eventID := stableIMEventID(topicName, tenantID, eventType, bizID, encoded)
+	envelope := kafkaEnvelope{
+		EventID:       eventID,
+		TenantCode:    tenantCode,
+		SourceService: "im-server",
+		TargetService: "im-server",
+		EventType:     eventType,
+		BizType:       bizType,
+		BizID:         bizID,
+		OccurredAt:    time.Now().UTC(),
+		Payload:       encoded,
+	}
+	if !actor.IsZero() {
+		envelope.Actor = &kafkaActor{Type: "user", ID: actor.UserId()}
+	}
+	return enqueueKafkaOutbox(tenantID, topicName, bizID, envelope)
+}
+
+func handleKafkaWorkbenchEvent(ctx context.Context, envelope kafkaEnvelope, _ kafka.Message) error {
+	tenantID, err := kafkaTenantID(envelope.TenantCode)
+	if err != nil {
+		return err
+	}
+	consume, claimed, err := beginKafkaConsumeForTopic(tenantID, envelope, "workbench_event")
+	if err != nil || !claimed {
+		return err
+	}
+	var payload kafkaWorkbenchPayload
+	if err = json.Unmarshal(envelope.Payload, &payload); err != nil {
+		return markKafkaDeadLetter(ctx, tenantID, envelope, consume, "MALFORMED_PAYLOAD", err.Error())
+	}
+	if err = executeKafkaWorkbenchEvent(tenantID, envelope, payload); err != nil {
+		failed, markErr := store.HTTPInfrastructure.MarkKafkaConsumeFailed(tenantID, envelope.EventID, kafkaBus.config.ConsumerGroup,
+			kafkaCommandErrorReason(err), err.Error(), time.Now().UTC().Round(time.Millisecond))
+		if markErr != nil {
+			return markErr
+		}
+		if failed != nil && failed.RetryCount >= kafkaBus.config.MaxRetryCount {
+			return markKafkaDeadLetter(ctx, tenantID, envelope, failed, kafkaCommandErrorReason(err), err.Error())
+		}
+		sleepKafkaRetryBackoff(failed.RetryCount)
+		return err
+	}
+	return store.HTTPInfrastructure.MarkKafkaConsumeSucceeded(tenantID, envelope.EventID, kafkaBus.config.ConsumerGroup, time.Now().UTC().Round(time.Millisecond))
+}
+
+func executeKafkaWorkbenchEvent(tenantID types.TenantID, envelope kafkaEnvelope, payload kafkaWorkbenchPayload) error {
+	actor := types.ParseUserId(payload.ActorUserID)
+	if actor.IsZero() {
+		actor = types.ParseUid(payload.ActorUserID)
+	}
+	for _, raw := range payload.TargetUserIDs {
+		target := types.ParseUserId(raw)
+		if target.IsZero() {
+			continue
+		}
+		eventID := stableIMEventID("workbench", envelope.EventID, target.UserId())
+		if target != actor && payload.Title != "" {
+			if err := createIMSystemNotification(tenantID, target, stableIMEventID(eventID, "notification"), payload.Action,
+				payload.Title, payload.Summary, types.KVMap{"module": payload.Module, "resource_id": payload.ResourceID}); err != nil {
+				return err
+			}
+		}
+		if err := enqueueIMBusinessEvent(tenantID, eventID, "user", target.UserId(), actor, payload.Action, payload.Data); err != nil {
+			return err
+		}
+	}
+	return nil
+}

server/main.go

@@ -324,6 +324,9 @@ type configType struct {
 	Cluster   json.RawMessage             `json:"cluster_config"`
 	Plugin    json.RawMessage             `json:"plugins"`
 	Store     json.RawMessage             `json:"store_config"`
+	Kafka     json.RawMessage             `json:"kafka"`
+	Internal  json.RawMessage             `json:"internal_api"`
+	AdminAPI  json.RawMessage             `json:"admin_internal_api"`
 	Push      json.RawMessage             `json:"push"`
 	TLS       json.RawMessage             `json:"tls"`
 	Auth      map[string]json.RawMessage  `json:"auth_config"`
@@ -450,6 +453,17 @@ func main() {
 	}()
 	statsRegisterDbStats()
 
+	if err = initKafkaRuntime(config.Kafka); err != nil {
+		logs.Err.Fatal("Failed to initialize Kafka:", err)
+	}
+	defer stopKafkaRuntime()
+	if err = initInternalAPI(config.Internal); err != nil {
+		logs.Err.Fatal("Failed to initialize internal API:", err)
+	}
+	if err = initAdminInternalAPI(config.AdminAPI); err != nil {
+		logs.Err.Fatal("Failed to initialize Admin internal API client:", err)
+	}
+
 	// API key signing secret
 	globals.apiKeySalt = config.APIKeySalt
 	err = store.InitAuthLogicalNames(config.Auth["logical_names"])
@@ -678,6 +692,13 @@ func main() {
 	// The hub (the main message router)
 	globals.hub = newHub()
 	startRedisGroupFanout(context.Background())
+	startKafkaAdminCommandConsumer()
+	startKafkaConfigEventConsumer()
+	startKafkaCacheInvalidateConsumer()
+	startKafkaMessagePersistConsumer()
+	startKafkaFileLifecycleConsumer()
+	startKafkaWorkbenchEventConsumer()
+	startKafkaOutboxPoller()
 	stopExternalBusinessEvents := startIMExternalBusinessEventPoller()
 	defer stopExternalBusinessEvents()
 
@@ -832,6 +853,7 @@ func main() {
 	mux.HandleFunc(config.ApiPath+"v1/im/search/conversations", serveIMConversationSearch)
 	mux.HandleFunc(config.ApiPath+"v1/im/conversations/p2p", serveIMP2PConversation)
 	mux.HandleFunc(config.ApiPath+"v1/im/forward-targets", serveIMForwardTargets)
+	mux.HandleFunc(config.ApiPath+"v1/im/messages/forward", serveIMMessageForward)
 	mux.HandleFunc(config.ApiPath+"v1/im/system-notifications", serveIMSystemNotifications)
 	mux.HandleFunc(config.ApiPath+"v1/im/system-notifications/", serveIMSystemNotificationRead)
 	mux.HandleFunc(config.ApiPath+"v1/im/favorites", serveIMFavorites)
@@ -852,6 +874,10 @@ func main() {
 	mux.HandleFunc(config.ApiPath+"v1/im/blocks/", serveIMBlockUser)
 	mux.HandleFunc(config.ApiPath+"v1/im/friends", serveIMFriends)
 	mux.HandleFunc(config.ApiPath+"v1/im/friends/", serveIMFriendResource)
+	adminQueryPrefix := config.ApiPath + "internal/admin-query/"
+	mux.HandleFunc(adminQueryPrefix, func(wrt http.ResponseWriter, req *http.Request) {
+		serveInternalAdminQuery(wrt, req, adminQueryPrefix)
+	})
 	// Handle long polling clients. Enable compression.
 	mux.Handle(config.ApiPath+"v0/channels/lp", gh.CompressHandler(http.HandlerFunc(serveLongPoll)))
 	if config.Media != nil {

server/session_test.go

@@ -27,6 +27,13 @@ func (s testTenantStore) GetByID(types.TenantID) (*types.Tenant, error) {
 	return s.tenant, s.err
 }
 
+func (s testTenantStore) ListActive(int) ([]types.Tenant, error) {
+	if s.err != nil || s.tenant == nil {
+		return nil, s.err
+	}
+	return []types.Tenant{*s.tenant}, nil
+}
+
 func test_makeSession(uid types.Uid) *Session {
 	return &Session{
 		send:         make(chan any, 10),

server/sessionstore.go

@@ -241,6 +241,33 @@ func (ss *SessionStore) EvictUser(uid types.Uid, skipSid string) {
 	statsSet("LiveSessions", int64(len(ss.sessCache)))
 }
 
+// EvictUserDevice terminates all sessions of a given user connected with a device.
+func (ss *SessionStore) EvictUserDevice(uid types.Uid, deviceID string) int {
+	ss.lock.Lock()
+	defer ss.lock.Unlock()
+
+	if uid.IsZero() || deviceID == "" {
+		return 0
+	}
+	evicted := NoErrEvicted("", "", types.TimeNow())
+	evicted.AsUser = uid.UserId()
+	count := 0
+	for _, s := range ss.sessCache {
+		if s.uid == uid && s.deviceID == deviceID && !s.isMultiplex() {
+			_, data := s.serialize(evicted)
+			s.stopSession(data)
+			delete(ss.sessCache, s.sid)
+			if s.proto == LPOLL {
+				ss.lru.Remove(s.lpTracker)
+			}
+			count++
+		}
+	}
+
+	statsSet("LiveSessions", int64(len(ss.sessCache)))
+	return count
+}
+
 // NodeRestarted removes stale sessions from a restarted cluster node.
 //   - nodeName is the name of affected node
 //   - fingerprint is the new fingerprint of the node.

server/store/config_cache.go

@@ -2,6 +2,7 @@ package store
 
 import (
 	"sort"
+	"strconv"
 	"strings"
 	"sync"
 	"time"
@@ -42,6 +43,18 @@ func (c *configCache[T]) set(key string, value T) {
 	c.mu.Unlock()
 }
 
+func (c *configCache[T]) delete(key string) {
+	c.mu.Lock()
+	delete(c.items, key)
+	c.mu.Unlock()
+}
+
+func (c *configCache[T]) clear() {
+	c.mu.Lock()
+	c.items = make(map[string]configCacheEntry[T])
+	c.mu.Unlock()
+}
+
 var (
 	tenantByCodeCache       = newConfigCache[*types.Tenant]()
 	tenantByIDCache         = newConfigCache[*types.Tenant]()
@@ -83,3 +96,43 @@ func thirdPartyCacheKey(group string, keys ...string) string {
 	sort.Strings(normalized)
 	return group + ":" + strings.Join(normalized, ",")
 }
+
+// InvalidateTenantConfigCache clears cached tenant registry/config records. An
+// empty selector clears all tenant records.
+func InvalidateTenantConfigCache(code string, id types.TenantID) {
+	if strings.TrimSpace(code) != "" {
+		tenantByCodeCache.delete(strings.TrimSpace(code))
+	} else {
+		tenantByCodeCache.clear()
+	}
+	if id.IsValid() {
+		tenantByIDCache.delete(formatTenantIDForCache(id))
+	} else if strings.TrimSpace(code) == "" {
+		tenantByIDCache.clear()
+	}
+}
+
+// InvalidateRegistrationConfigCache clears cached global registration config.
+func InvalidateRegistrationConfigCache() {
+	registrationSMSCache.clear()
+}
+
+// InvalidateThirdPartyConfigCache clears cached third-party config.
+func InvalidateThirdPartyConfigCache(group string) {
+	if strings.TrimSpace(group) == "" {
+		thirdPartyConfigCache.clear()
+		return
+	}
+	prefix := strings.TrimSpace(group) + ":"
+	thirdPartyConfigCache.mu.Lock()
+	for key := range thirdPartyConfigCache.items {
+		if strings.HasPrefix(key, prefix) {
+			delete(thirdPartyConfigCache.items, key)
+		}
+	}
+	thirdPartyConfigCache.mu.Unlock()
+}
+
+func formatTenantIDForCache(id types.TenantID) string {
+	return strconv.FormatInt(int64(id), 10)
+}

server/store/group_message_runtime.go

@@ -69,6 +69,7 @@ type groupMessageRuntime interface {
 	RemoveCachedSetMember(tenantID types.TenantID, namespace, id, member string) error
 	CachedSetContains(tenantID types.TenantID, namespace, id, member string) (bool, error)
 	AllowRate(tenantID types.TenantID, scope string, limit int, window time.Duration) (bool, int, time.Duration, error)
+	ClaimShortState(namespace, id string, ttl time.Duration) (bool, error)
 }
 
 var groupMsgRuntime groupMessageRuntime
@@ -88,6 +89,14 @@ func MarkUserOnline(tenantID types.TenantID, uid types.Uid, sid, nodeID, device,
 	return groupMsgRuntime.MarkUserOnline(tenantID, uid, sid, nodeID, device, userAgent)
 }
 
+func ClaimShortState(namespace, id string, ttl time.Duration) (bool, bool, error) {
+	if groupMsgRuntime == nil || namespace == "" || id == "" {
+		return false, false, nil
+	}
+	ok, err := groupMsgRuntime.ClaimShortState(namespace, id, ttl)
+	return ok, true, err
+}
+
 func RefreshUserPresence(tenantID types.TenantID, uid types.Uid, sid, nodeID string) error {
 	if groupMsgRuntime == nil || !tenantID.IsValid() || uid.IsZero() || sid == "" {
 		return nil
@@ -169,6 +178,13 @@ func DeleteCachedObject(tenantID types.TenantID, namespace, id string) error {
 	return groupMsgRuntime.DeleteCachedObject(tenantID, namespace, id)
 }
 
+func InvalidateHotMessages(tenantID types.TenantID, topic string, ranges []types.Range) error {
+	if groupMsgRuntime == nil || !tenantID.IsValid() || topic == "" {
+		return nil
+	}
+	return groupMsgRuntime.InvalidateHotMessages(tenantID, topic, ranges)
+}
+
 func AddCachedSetMember(tenantID types.TenantID, namespace, id, member string) error {
 	if groupMsgRuntime == nil || !tenantID.IsValid() || namespace == "" || id == "" || member == "" {
 		return nil
@@ -730,6 +746,13 @@ func (r *redisGroupMessageRuntime) AllowRate(tenantID types.TenantID, scope stri
 	return count <= int64(limit), int(count), ttl, nil
 }
 
+func (r *redisGroupMessageRuntime) ClaimShortState(namespace, id string, ttl time.Duration) (bool, error) {
+	if ttl <= 0 {
+		ttl = time.Duration(r.cfg.ShortTTL) * time.Second
+	}
+	return r.client.SetNX(context.Background(), r.shortStateKey(namespace, id), "1", ttl).Result()
+}
+
 func (r *redisGroupMessageRuntime) hotSeqIDs(tenantID types.TenantID, topic string, opts *types.QueryOpt, limit int) ([]int64, bool, error) {
 	if opts != nil && len(opts.IdRanges) > 0 {
 		seqs := expandRanges(opts.IdRanges, limit)
@@ -851,6 +874,10 @@ func (r *redisGroupMessageRuntime) rateKey(tenantID types.TenantID, scope string
 	return r.key(tenantID, "rate", scope)
 }
 
+func (r *redisGroupMessageRuntime) shortStateKey(namespace, id string) string {
+	return r.key(types.ZeroTenantID, "short", namespace, id)
+}
+
 func (r *redisGroupMessageRuntime) cacheTTL(namespace string) time.Duration {
 	ttl := r.cfg.CacheTTL
 	if namespace == "verification" || namespace == "action_token" {

server/store/infrastructure.go

@@ -14,6 +14,18 @@ type HTTPInfrastructurePersistenceInterface interface {
 	CreateEvent(types.TenantID, *types.BusinessEvent) error
 	ListPendingEvents(types.TenantID, int, time.Time) ([]types.BusinessEvent, error)
 	MarkEventDelivered(types.TenantID, string, time.Time) error
+	BeginKafkaConsume(types.TenantID, *types.KafkaConsumeRecord, time.Time) (*types.KafkaConsumeRecord, bool, error)
+	MarkKafkaConsumeSucceeded(types.TenantID, string, string, time.Time) error
+	MarkKafkaConsumeFailed(types.TenantID, string, string, string, string, time.Time) (*types.KafkaConsumeRecord, error)
+	MarkKafkaConsumeDead(types.TenantID, *types.KafkaDeadLetter, time.Time) error
+	ListKafkaDeadLetters(types.TenantID, types.KafkaDeadLetterListOptions) (*types.KafkaDeadLetterPage, error)
+	GetKafkaDeadLetter(types.TenantID, string) (*types.KafkaDeadLetter, error)
+	MarkKafkaDeadLetter(types.TenantID, string, string, string, string, time.Time) error
+	MarkKafkaDeadLetterReplay(types.TenantID, string, string, string, time.Time) (*types.KafkaDeadLetter, error)
+	CreateKafkaOutbox(types.TenantID, *types.KafkaOutboxRecord) error
+	ListPendingKafkaOutbox(types.TenantID, int, time.Time) ([]types.KafkaOutboxRecord, error)
+	MarkKafkaOutboxSent(types.TenantID, string, time.Time) error
+	MarkKafkaOutboxFailed(types.TenantID, string, string, string, time.Time, time.Time) error
 }
 
 type httpInfrastructureMapper struct{}
@@ -79,3 +91,99 @@ func (httpInfrastructureMapper) MarkEventDelivered(tenantID types.TenantID, even
 	}
 	return infrastructure.TenantBusinessEventMarkDelivered(tenantID, eventID, now)
 }
+
+func (httpInfrastructureMapper) BeginKafkaConsume(tenantID types.TenantID, record *types.KafkaConsumeRecord, deadline time.Time) (*types.KafkaConsumeRecord, bool, error) {
+	infrastructure, err := tenantHTTPInfrastructureAdapter(tenantID)
+	if err != nil {
+		return nil, false, err
+	}
+	return infrastructure.TenantKafkaConsumeBegin(tenantID, record, deadline)
+}
+
+func (httpInfrastructureMapper) MarkKafkaConsumeSucceeded(tenantID types.TenantID, eventID, consumerGroup string, now time.Time) error {
+	infrastructure, err := tenantHTTPInfrastructureAdapter(tenantID)
+	if err != nil {
+		return err
+	}
+	return infrastructure.TenantKafkaConsumeSucceed(tenantID, eventID, consumerGroup, now)
+}
+
+func (httpInfrastructureMapper) MarkKafkaConsumeFailed(tenantID types.TenantID, eventID, consumerGroup, reason, message string, now time.Time) (*types.KafkaConsumeRecord, error) {
+	infrastructure, err := tenantHTTPInfrastructureAdapter(tenantID)
+	if err != nil {
+		return nil, err
+	}
+	return infrastructure.TenantKafkaConsumeFail(tenantID, eventID, consumerGroup, reason, message, now)
+}
+
+func (httpInfrastructureMapper) MarkKafkaConsumeDead(tenantID types.TenantID, dead *types.KafkaDeadLetter, now time.Time) error {
+	infrastructure, err := tenantHTTPInfrastructureAdapter(tenantID)
+	if err != nil {
+		return err
+	}
+	return infrastructure.TenantKafkaConsumeDead(tenantID, dead, now)
+}
+
+func (httpInfrastructureMapper) ListKafkaDeadLetters(tenantID types.TenantID, opts types.KafkaDeadLetterListOptions) (*types.KafkaDeadLetterPage, error) {
+	infrastructure, err := tenantHTTPInfrastructureAdapter(tenantID)
+	if err != nil {
+		return nil, err
+	}
+	return infrastructure.TenantKafkaDeadLetterList(tenantID, opts)
+}
+
+func (httpInfrastructureMapper) GetKafkaDeadLetter(tenantID types.TenantID, deadID string) (*types.KafkaDeadLetter, error) {
+	infrastructure, err := tenantHTTPInfrastructureAdapter(tenantID)
+	if err != nil {
+		return nil, err
+	}
+	return infrastructure.TenantKafkaDeadLetterGet(tenantID, deadID)
+}
+
+func (httpInfrastructureMapper) MarkKafkaDeadLetter(tenantID types.TenantID, deadID, status, resolvedBy, note string, now time.Time) error {
+	infrastructure, err := tenantHTTPInfrastructureAdapter(tenantID)
+	if err != nil {
+		return err
+	}
+	return infrastructure.TenantKafkaDeadLetterMark(tenantID, deadID, status, resolvedBy, note, now)
+}
+
+func (httpInfrastructureMapper) MarkKafkaDeadLetterReplay(tenantID types.TenantID, deadID, resolvedBy, note string, now time.Time) (*types.KafkaDeadLetter, error) {
+	infrastructure, err := tenantHTTPInfrastructureAdapter(tenantID)
+	if err != nil {
+		return nil, err
+	}
+	return infrastructure.TenantKafkaDeadLetterMarkReplay(tenantID, deadID, resolvedBy, note, now)
+}
+
+func (httpInfrastructureMapper) CreateKafkaOutbox(tenantID types.TenantID, record *types.KafkaOutboxRecord) error {
+	infrastructure, err := tenantHTTPInfrastructureAdapter(tenantID)
+	if err != nil {
+		return err
+	}
+	return infrastructure.TenantKafkaOutboxCreate(tenantID, record)
+}
+
+func (httpInfrastructureMapper) ListPendingKafkaOutbox(tenantID types.TenantID, limit int, now time.Time) ([]types.KafkaOutboxRecord, error) {
+	infrastructure, err := tenantHTTPInfrastructureAdapter(tenantID)
+	if err != nil {
+		return nil, err
+	}
+	return infrastructure.TenantKafkaOutboxListPending(tenantID, limit, now)
+}
+
+func (httpInfrastructureMapper) MarkKafkaOutboxSent(tenantID types.TenantID, eventID string, now time.Time) error {
+	infrastructure, err := tenantHTTPInfrastructureAdapter(tenantID)
+	if err != nil {
+		return err
+	}
+	return infrastructure.TenantKafkaOutboxMarkSent(tenantID, eventID, now)
+}
+
+func (httpInfrastructureMapper) MarkKafkaOutboxFailed(tenantID types.TenantID, eventID, reason, message string, nextRetryAt, now time.Time) error {
+	infrastructure, err := tenantHTTPInfrastructureAdapter(tenantID)
+	if err != nil {
+		return err
+	}
+	return infrastructure.TenantKafkaOutboxMarkFailed(tenantID, eventID, reason, message, nextRetryAt, now)
+}

server/store/internal_admin_query.go

@@ -0,0 +1,43 @@
+package store
+
+import (
+	adapter "github.com/tinode/chat/server/db"
+	"github.com/tinode/chat/server/store/types"
+)
+
+type InternalAdminQueryPersistenceInterface interface {
+	Query(types.TenantID, string, string, []any, int, int) (*types.InternalAdminQueryResult, error)
+	QueryOne(types.TenantID, string, []any) (map[string]any, error)
+}
+
+type internalAdminQueryMapper struct{}
+
+var InternalAdminQueries InternalAdminQueryPersistenceInterface = internalAdminQueryMapper{}
+
+func internalAdminQueryAdapter(tenantID types.TenantID) (adapter.TenantInternalAdminQueryAdapter, error) {
+	business, err := tenantBusinessAdapter(tenantID)
+	if err != nil {
+		return nil, err
+	}
+	queryAdp, ok := business.(adapter.TenantInternalAdminQueryAdapter)
+	if !ok {
+		return nil, types.ErrUnsupported
+	}
+	return queryAdp, nil
+}
+
+func (internalAdminQueryMapper) Query(tenantID types.TenantID, query string, countQuery string, args []any, page, perPage int) (*types.InternalAdminQueryResult, error) {
+	queryAdp, err := internalAdminQueryAdapter(tenantID)
+	if err != nil {
+		return nil, err
+	}
+	return queryAdp.TenantInternalAdminQuery(tenantID, query, countQuery, args, page, perPage)
+}
+
+func (internalAdminQueryMapper) QueryOne(tenantID types.TenantID, query string, args []any) (map[string]any, error) {
+	queryAdp, err := internalAdminQueryAdapter(tenantID)
+	if err != nil {
+		return nil, err
+	}
+	return queryAdp.TenantInternalAdminQueryOne(tenantID, query, args)
+}

server/store/mock_store/mock_store.go

@@ -330,6 +330,36 @@ func (mr *MockTenantsPersistenceInterfaceMockRecorder) GetByCode(code interface{
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetByCode", reflect.TypeOf((*MockTenantsPersistenceInterface)(nil).GetByCode), code)
 }
 
+// GetByID mocks base method.
+func (m *MockTenantsPersistenceInterface) GetByID(id types.TenantID) (*types.Tenant, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "GetByID", id)
+	ret0, _ := ret[0].(*types.Tenant)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// GetByID indicates an expected call of GetByID.
+func (mr *MockTenantsPersistenceInterfaceMockRecorder) GetByID(id interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetByID", reflect.TypeOf((*MockTenantsPersistenceInterface)(nil).GetByID), id)
+}
+
+// ListActive mocks base method.
+func (m *MockTenantsPersistenceInterface) ListActive(limit int) ([]types.Tenant, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "ListActive", limit)
+	ret0, _ := ret[0].([]types.Tenant)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// ListActive indicates an expected call of ListActive.
+func (mr *MockTenantsPersistenceInterfaceMockRecorder) ListActive(limit interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListActive", reflect.TypeOf((*MockTenantsPersistenceInterface)(nil).ListActive), limit)
+}
+
 // MockUsersPersistenceInterface is a mock of UsersPersistenceInterface interface.
 type MockUsersPersistenceInterface struct {
 	ctrl     *gomock.Controller
@@ -1220,6 +1250,36 @@ func (mr *MockMessagesPersistenceInterfaceMockRecorder) Save(tenantID, msg, atta
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Save", reflect.TypeOf((*MockMessagesPersistenceInterface)(nil).Save), tenantID, msg, attachmentURLs, readBySender)
 }
 
+// PreparePersist mocks base method.
+func (m *MockMessagesPersistenceInterface) PreparePersist(tenantID types.TenantID, msg *types.Message, attachmentURLs []string, readBySender bool) (*types.MessageSaveResult, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "PreparePersist", tenantID, msg, attachmentURLs, readBySender)
+	ret0, _ := ret[0].(*types.MessageSaveResult)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// PreparePersist indicates an expected call of PreparePersist.
+func (mr *MockMessagesPersistenceInterfaceMockRecorder) PreparePersist(tenantID, msg, attachmentURLs, readBySender interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PreparePersist", reflect.TypeOf((*MockMessagesPersistenceInterface)(nil).PreparePersist), tenantID, msg, attachmentURLs, readBySender)
+}
+
+// PersistPrepared mocks base method.
+func (m *MockMessagesPersistenceInterface) PersistPrepared(tenantID types.TenantID, msg *types.Message, attachmentURLs []string, readBySender bool) (*types.MessageSaveResult, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "PersistPrepared", tenantID, msg, attachmentURLs, readBySender)
+	ret0, _ := ret[0].(*types.MessageSaveResult)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// PersistPrepared indicates an expected call of PersistPrepared.
+func (mr *MockMessagesPersistenceInterfaceMockRecorder) PersistPrepared(tenantID, msg, attachmentURLs, readBySender interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PersistPrepared", reflect.TypeOf((*MockMessagesPersistenceInterface)(nil).PersistPrepared), tenantID, msg, attachmentURLs, readBySender)
+}
+
 // MockDevicePersistenceInterface is a mock of DevicePersistenceInterface interface.
 type MockDevicePersistenceInterface struct {
 	ctrl     *gomock.Controller

server/store/store.go

@@ -151,6 +151,7 @@ const (
 type TenantsPersistenceInterface interface {
 	GetByCode(code string) (*types.Tenant, error)
 	GetByID(id types.TenantID) (*types.Tenant, error)
+	ListActive(limit int) ([]types.Tenant, error)
 }
 
 type tenantsMapper struct{}
@@ -158,6 +159,22 @@ type tenantsMapper struct{}
 // Tenants is the tenant registry access object.
 var Tenants TenantsPersistenceInterface
 
+// AdminInternalConfigProvider optionally resolves hotgo-owned configuration
+// through the Admin service instead of reading the hotgo database directly.
+type AdminInternalConfigProvider interface {
+	GetTenantByCode(code string) (*types.Tenant, error)
+	GetTenantByID(id types.TenantID) (*types.Tenant, error)
+	ListActiveTenants(limit int) ([]types.Tenant, error)
+	RegistrationSMSEnabled() (bool, error)
+	ThirdPartyConfig(group string, keys ...string) (map[string]string, error)
+}
+
+var adminInternalConfigProvider AdminInternalConfigProvider
+
+func SetAdminInternalConfigProvider(provider AdminInternalConfigProvider) {
+	adminInternalConfigProvider = provider
+}
+
 // RegistrationConfigPersistenceInterface exposes global registration policy.
 type RegistrationConfigPersistenceInterface interface {
 	SMSEnabled() (bool, error)
@@ -172,6 +189,14 @@ func (registrationConfigMapper) SMSEnabled() (bool, error) {
 	if enabled, ok := registrationSMSCache.get("im:register_sms_enabled"); ok {
 		return enabled, nil
 	}
+	if adminInternalConfigProvider != nil {
+		if enabled, err := adminInternalConfigProvider.RegistrationSMSEnabled(); err == nil {
+			registrationSMSCache.set("im:register_sms_enabled", enabled)
+			return enabled, nil
+		} else {
+			logs.Warn.Printf("admin internal registration config fallback failed: %v", err)
+		}
+	}
 	configAdp, ok := adp.(adapter.RegistrationConfigAdapter)
 	if !ok {
 		return true, types.ErrUnsupported
@@ -198,6 +223,14 @@ func (thirdPartyConfigMapper) Get(group string, keys ...string) (map[string]stri
 	if cached, ok := thirdPartyConfigCache.get(cacheKey); ok {
 		return cloneStringMap(cached), nil
 	}
+	if adminInternalConfigProvider != nil {
+		if result, err := adminInternalConfigProvider.ThirdPartyConfig(group, keys...); err == nil {
+			thirdPartyConfigCache.set(cacheKey, cloneStringMap(result))
+			return result, nil
+		} else {
+			logs.Warn.Printf("admin internal third-party config fallback failed: group=%s err=%v", group, err)
+		}
+	}
 	configAdp, ok := adp.(adapter.ThirdPartyConfigAdapter)
 	if !ok {
 		return nil, types.ErrUnsupported
@@ -261,6 +294,15 @@ func (tenantsMapper) GetByCode(code string) (*types.Tenant, error) {
 	if tenant, ok := tenantByCodeCache.get(code); ok {
 		return cloneTenant(tenant), nil
 	}
+	if adminInternalConfigProvider != nil {
+		if tenant, err := adminInternalConfigProvider.GetTenantByCode(code); err == nil && tenant != nil {
+			tenantByCodeCache.set(code, cloneTenant(tenant))
+			tenantByIDCache.set(strconv.FormatInt(int64(tenant.ID), 10), cloneTenant(tenant))
+			return tenant, nil
+		} else if err != nil {
+			logs.Warn.Printf("admin internal tenant by code fallback failed: code=%s err=%v", code, err)
+		}
+	}
 	tenantAdp, ok := adp.(adapter.TenantAdapter)
 	if !ok {
 		return nil, types.ErrUnsupported
@@ -282,6 +324,15 @@ func (tenantsMapper) GetByID(id types.TenantID) (*types.Tenant, error) {
 	if tenant, ok := tenantByIDCache.get(idKey); ok {
 		return cloneTenant(tenant), nil
 	}
+	if adminInternalConfigProvider != nil {
+		if tenant, err := adminInternalConfigProvider.GetTenantByID(id); err == nil && tenant != nil {
+			tenantByIDCache.set(idKey, cloneTenant(tenant))
+			tenantByCodeCache.set(tenant.Code, cloneTenant(tenant))
+			return tenant, nil
+		} else if err != nil {
+			logs.Warn.Printf("admin internal tenant by id fallback failed: tenant=%d err=%v", id, err)
+		}
+	}
 	tenantAdp, ok := adp.(adapter.TenantAdapter)
 	if !ok {
 		return nil, types.ErrUnsupported
@@ -294,6 +345,29 @@ func (tenantsMapper) GetByID(id types.TenantID) (*types.Tenant, error) {
 	return tenant, err
 }
 
+func (tenantsMapper) ListActive(limit int) ([]types.Tenant, error) {
+	if limit <= 0 {
+		limit = 1000
+	}
+	if adminInternalConfigProvider != nil {
+		if tenants, err := adminInternalConfigProvider.ListActiveTenants(limit); err == nil {
+			for i := range tenants {
+				tenant := tenants[i]
+				tenantByIDCache.set(strconv.FormatInt(int64(tenant.ID), 10), cloneTenant(&tenant))
+				tenantByCodeCache.set(tenant.Code, cloneTenant(&tenant))
+			}
+			return tenants, nil
+		} else {
+			logs.Warn.Printf("admin internal active tenants fallback failed: err=%v", err)
+		}
+	}
+	tenantAdp, ok := adp.(adapter.TenantAdapter)
+	if !ok {
+		return nil, types.ErrUnsupported
+	}
+	return tenantAdp.TenantListActive(limit)
+}
+
 func tenantAuthAdapter(tenantID types.TenantID) (adapter.TenantAuthAdapter, error) {
 	if !tenantID.IsValid() {
 		return nil, types.ErrMalformed
@@ -1267,6 +1341,8 @@ func subscriptionCacheID(topic string, user types.Uid, keepDeleted bool) string
 // MessagesPersistenceInterface is an interface which defines methods for persistent storage of messages.
 type MessagesPersistenceInterface interface {
 	Save(tenantID types.TenantID, msg *types.Message, attachmentURLs []string, readBySender bool) (*types.MessageSaveResult, error)
+	PreparePersist(tenantID types.TenantID, msg *types.Message, attachmentURLs []string, readBySender bool) (*types.MessageSaveResult, error)
+	PersistPrepared(tenantID types.TenantID, msg *types.Message, attachmentURLs []string, readBySender bool) (*types.MessageSaveResult, error)
 	GetOne(tenantID types.TenantID, topic string, seq int64) (*types.Message, error)
 	Recall(tenantID types.TenantID, topic string, seq int64, actor types.Uid, expectedVersion int, recalledAt time.Time) (*types.Message, error)
 	Edit(tenantID types.TenantID, topic string, seq int64, actor types.Uid, expectedVersion int, head types.KVMap, content any, reason string, editedAt time.Time) (*types.Message, error)
@@ -3063,6 +3139,17 @@ func cachedBlockContains(tenantID types.TenantID, blocker, blocked types.Uid) (b
 
 // Save message
 func (messagesMapper) Save(tenantID types.TenantID, msg *types.Message, attachmentURLs []string, readBySender bool) (*types.MessageSaveResult, error) {
+	saveResult, err := Messages.PreparePersist(tenantID, msg, attachmentURLs, readBySender)
+	if err != nil {
+		return nil, err
+	}
+	if saveResult != nil && saveResult.Duplicate {
+		return saveResult, nil
+	}
+	return Messages.PersistPrepared(tenantID, msg, attachmentURLs, readBySender)
+}
+
+func (messagesMapper) PreparePersist(tenantID types.TenantID, msg *types.Message, _ []string, _ bool) (*types.MessageSaveResult, error) {
 	if msg == nil || msg.TenantID != tenantID {
 		return nil, types.ErrMalformed
 	}
@@ -3126,12 +3213,29 @@ func (messagesMapper) Save(tenantID types.TenantID, msg *types.Message, attachme
 	}
 	msg.InitTimes()
 	msg.SetUid(Store.GetUid())
-	// Increment topic's or user's SeqId
+	// Reserve topic's or user's SeqId before optional async persistence.
 	err = tenantAdp.TenantTopicUpdateOnMessage(tenantID, msg.Topic, msg)
 	if err != nil {
 		return nil, err
 	}
+	return &types.MessageSaveResult{SeqId: msg.SeqId}, nil
+}
 
+func (messagesMapper) PersistPrepared(tenantID types.TenantID, msg *types.Message, attachmentURLs []string, readBySender bool) (*types.MessageSaveResult, error) {
+	if msg == nil || msg.TenantID != tenantID || msg.Topic == "" || msg.SeqId <= 0 {
+		return nil, types.ErrMalformed
+	}
+	tenantAdp, err := tenantBusinessAdapter(tenantID)
+	if err != nil {
+		return nil, err
+	}
+	clientMsgID, err := normalizeClientMsgID(msg.Head)
+	if err != nil {
+		return nil, err
+	}
+	msg.ClientMsgID = clientMsgID
+	fromUid := types.ParseUid(msg.From)
+	isGroupMessage := types.GetTopicCat(msg.Topic) == types.TopicCatGrp
 	if isGroupMessage {
 		err = tenantAdp.TenantGroupMessageSave(tenantID, msg)
 	} else {
@@ -3220,6 +3324,14 @@ func (messagesMapper) Save(tenantID types.TenantID, msg *types.Message, attachme
 	return &types.MessageSaveResult{SeqId: msg.SeqId, MarkedReadBySender: markedReadBySender}, nil
 }
 
+func PrepareMessagePersist(tenantID types.TenantID, msg *types.Message, attachmentURLs []string, readBySender bool) (*types.MessageSaveResult, error) {
+	return Messages.PreparePersist(tenantID, msg, attachmentURLs, readBySender)
+}
+
+func PersistPreparedMessage(tenantID types.TenantID, msg *types.Message, attachmentURLs []string, readBySender bool) (*types.MessageSaveResult, error) {
+	return Messages.PersistPrepared(tenantID, msg, attachmentURLs, readBySender)
+}
+
 func retryTenantMessageSaveAfterSeqConflict(tenantAdp adapter.TenantBusinessAdapter, tenantID types.TenantID, msg *types.Message) error {
 	latest, err := tenantAdp.TenantMessageGetAll(tenantID, msg.Topic, types.ZeroUid, &types.QueryOpt{Before: 1 << 62, Limit: 1})
 	if err != nil {
@@ -3821,7 +3933,7 @@ func (fileMapper) LinkAttachments(tenantID types.TenantID, topic string, msgId t
 
 	if len(fids) > 0 {
 		userId := types.ZeroUid
-		if types.GetTopicCat(topic) == types.TopicCatMe {
+		if topic != "" && types.GetTopicCat(topic) == types.TopicCatMe {
 			userId = types.ParseUserId(topic)
 			topic = ""
 		}
@@ -4353,6 +4465,7 @@ func init() {
 	Tenants = tenantsMapper{}
 	RegistrationConfig = registrationConfigMapper{}
 	TenantEntry = tenantEntryMapper{}
+	InternalAdminQueries = internalAdminQueryMapper{}
 	Users = usersMapper{}
 	Topics = topicsMapper{}
 	Subs = subsMapper{}

server/store/types/infrastructure.go

@@ -12,6 +12,21 @@ const (
 
 	BusinessEventPending   = "pending"
 	BusinessEventDelivered = "delivered"
+
+	KafkaConsumeProcessing = "processing"
+	KafkaConsumeSucceeded  = "succeeded"
+	KafkaConsumeFailed     = "failed"
+	KafkaConsumeDead       = "dead"
+
+	KafkaDeadStatusDead          = "dead"
+	KafkaDeadStatusPendingReplay = "pending_replay"
+	KafkaDeadStatusReplayed      = "replayed"
+	KafkaDeadStatusIgnored       = "ignored"
+	KafkaDeadStatusResolved      = "resolved"
+
+	KafkaOutboxPending = "pending"
+	KafkaOutboxSent    = "sent"
+	KafkaOutboxFailed  = "failed"
 )
 
 // HTTPIdempotencyRecord stores the first response for one authenticated write.
@@ -46,6 +61,80 @@ type BusinessEvent struct {
 	UpdatedAt   time.Time
 }
 
+// KafkaConsumeRecord stores idempotent consumer state for one event and group.
+type KafkaConsumeRecord struct {
+	EventID            string
+	ConsumerGroup      string
+	SourceTopic        string
+	EventType          string
+	BizType            string
+	BizID              string
+	Status             string
+	RetryCount         int
+	LastReason         string
+	LastError          string
+	ProcessingDeadline *time.Time
+	SucceededAt        *time.Time
+	DeadAt             *time.Time
+	CreatedAt          time.Time
+	UpdatedAt          time.Time
+}
+
+// KafkaDeadLetter stores a poison Kafka event for manual handling.
+type KafkaDeadLetter struct {
+	ID            int64
+	DeadID        string
+	EventID       string
+	ParentEventID string
+	SourceTopic   string
+	ConsumerGroup string
+	EventType     string
+	BizType       string
+	BizID         string
+	Envelope      []byte
+	RetryCount    int
+	LastReason    string
+	LastError     string
+	Status        string
+	ReplayCount   int
+	CreatedAt     time.Time
+	UpdatedAt     time.Time
+}
+
+type KafkaDeadLetterListOptions struct {
+	Status    string
+	BizType   string
+	BizID     string
+	EventType string
+	Cursor    string
+	Limit     int
+}
+
+type KafkaDeadLetterPage struct {
+	Items      []KafkaDeadLetter
+	NextCursor string
+}
+
+// KafkaOutboxRecord stores a Kafka event waiting for reliable publication.
+type KafkaOutboxRecord struct {
+	ID          int64
+	EventID     string
+	Topic       string
+	EventType   string
+	BizType     string
+	BizID       string
+	Key         string
+	Envelope    []byte
+	Status      string
+	Attempts    int
+	NextRetryAt *time.Time
+	SentAt      *time.Time
+	LastReason  string
+	LastError   string
+	CreatedAt   time.Time
+	UpdatedAt   time.Time
+}
+
 // StableEventID derives an idempotent event identifier from domain identity.
 func StableEventID(parts ...any) string {
 	hash := sha256.New()

server/store/types/internal_admin_query.go

@@ -0,0 +1,8 @@
+package types
+
+type InternalAdminQueryResult struct {
+	Items   []map[string]any
+	Total   int64
+	Page    int
+	PerPage int
+}

server/tinode.conf

@@ -106,6 +106,53 @@
     }
   },
 
+  "kafka": {
+    "enabled": false,
+    "brokers": [],
+    "client_id": "im-server",
+    "consumer_group": "im-server",
+    "dial_timeout_ms": 3000,
+    "write_timeout_ms": 3000,
+    "read_timeout_ms": 3000,
+    "required_acks": 1,
+    "allow_auto_topic_creation": false,
+    "tls_enabled": false,
+    "max_retry_count": 5,
+    "retry_backoff_ms": [10000, 30000, 60000, 300000, 900000],
+    "dead_letter_retention_seconds": 7776000,
+    "topics": {
+      "admin_command": "im.admin.command",
+      "admin_command_result": "im.admin.command.result",
+      "config_event": "im.config.event",
+      "business_event": "im.business.event",
+      "message_persist": "im.message.persist",
+      "message_push": "im.message.push",
+      "file_security": "im.file.security",
+      "file_lifecycle": "im.file.lifecycle",
+      "workbench_event": "im.workbench.event",
+      "cache_invalidate": "im.cache.invalidate",
+      "audit_event": "im.audit.event",
+      "dead_letter": "im.dead.letter"
+    }
+  },
+  "internal_api": {
+    "enabled": false,
+    "time_window_seconds": 300,
+    "services": {
+      "hotgo-admin": {
+        "secret": "",
+        "allow": ["admin_query"]
+      }
+    }
+  },
+  "admin_internal_api": {
+    "enabled": false,
+    "base_url": "",
+    "service_name": "im-server",
+    "secret": "",
+    "timeout_ms": 3000
+  },
+
   "acc_validation": {
     "email": {
       "add_to_tags": false,

server/topic.go

@@ -9,6 +9,7 @@
 package main
 
 import (
+	"context"
 	"errors"
 	"sort"
 	"strings"
@@ -1104,7 +1105,7 @@ func (t *Topic) saveAndBroadcastMessage(msg *ClientComMessage, asUid types.Uid,
 		}
 	}
 
-	saveResult, err := store.Messages.Save(t.tenantID,
+	saveResult, err := saveMessageMaybeKafka(context.Background(), t.tenantID,
 		&types.Message{
 			ObjHeader: types.ObjHeader{CreatedAt: msg.Timestamp},
 			TenantID:  t.tenantID,