pro/im_server
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: complete shared database tenant sharding

Browse Source
This commit is contained in:
richjohnrambo
2026-09-23 02:19:07 +08:00
parent 1ec2feee47
commit e37b52c3c0
37 changed files with 946 additions and 1420 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/db/mysql/tenant_infrastructure.go
+5 -282
View File
@@ -4,15 +4,12 @@
package mysql
import (
"context"
"database/sql"
"encoding/json"
"strconv"
"strings"
"time"
"github.com/jmoiron/sqlx"
"github.com/tinode/chat/server/store"
t "github.com/tinode/chat/server/store/types"
)
@@ -26,214 +23,6 @@ func (a *adapter) infrastructureAdapter(id t.TenantID) (*adapter, error) {
return a, nil
}
func (a *adapter) TenantHTTPIdempotencyBegin(id t.TenantID, record *t.HTTPIdempotencyRecord) (*t.HTTPIdempotencyRecord, bool, error) {
tenantAdp, err := a.infrastructureAdapter(id)
if err != nil {
return nil, false, err
}
if record == nil || record.UserID.IsZero() || record.Method == "" || record.Route == "" || record.Key == "" || record.BodyHash == "" {
return nil, false, t.ErrMalformed
}
if tenantAdp != a {
return tenantAdp.TenantHTTPIdempotencyBegin(id, record)
}
now := t.TimeNow()
if record.ExpiresAt.IsZero() {
record.ExpiresAt = now.Add(24 * time.Hour)
}
_, err = a.db.Exec(`INSERT INTO im_http_idempotency
(tenant_id,user_id,method,route,idempotency_key,body_hash,state,response_status,response_body,expires_at,created_at,updated_at)
VALUES(?,?,?,?,?,?,?,?,?,?,?,?)`, id, decodeUID(record.UserID), record.Method, record.Route, record.Key,
record.BodyHash, t.HTTPIdempotencyProcessing, 0, nil, record.ExpiresAt, now, now)
if err == nil {
record.State = t.HTTPIdempotencyProcessing
record.CreatedAt = now
record.UpdatedAt = now
return record, true, nil
}
if !isDupe(err) {
return nil, false, err
}
existing, err := a.httpIdempotencyGet(id, record.UserID, record.Method, record.Route, record.Key)
if err == nil && !existing.ExpiresAt.After(now) {
res, deleteErr := a.db.Exec(`DELETE FROM im_http_idempotency
WHERE tenant_id=? AND user_id=? AND method=? AND route=? AND idempotency_key=? AND expires_at<=?`,
id, decodeUID(record.UserID), record.Method, record.Route, record.Key, now)
if deleteErr != nil {
return nil, false, deleteErr
}
if deleted, _ := res.RowsAffected(); deleted > 0 {
return a.TenantHTTPIdempotencyBegin(id, record)
}
current, getErr := a.httpIdempotencyGet(id, record.UserID, record.Method, record.Route, record.Key)
return current, false, getErr
}
return existing, false, err
}
func (a *adapter) TenantHTTPIdempotencyComplete(id t.TenantID, record *t.HTTPIdempotencyRecord) error {
tenantAdp, err := a.infrastructureAdapter(id)
if err != nil {
return err
}
if record == nil || record.UserID.IsZero() {
return t.ErrMalformed
}
if tenantAdp != a {
return tenantAdp.TenantHTTPIdempotencyComplete(id, record)
}
now := t.TimeNow()
res, err := a.db.Exec(`UPDATE im_http_idempotency SET state=?,response_status=?,response_body=?,updated_at=?
WHERE tenant_id=? AND user_id=? AND method=? AND route=? AND idempotency_key=? AND body_hash=? AND state=?`,
t.HTTPIdempotencyCompleted, record.ResponseStatus, record.ResponseBody, now, id, decodeUID(record.UserID),
record.Method, record.Route, record.Key, record.BodyHash, t.HTTPIdempotencyProcessing)
if err != nil {
return err
}
if count, _ := res.RowsAffected(); count == 0 {
return t.ErrNotFound
}
return nil
}
func (a *adapter) TenantHTTPIdempotencyAbort(id t.TenantID, record *t.HTTPIdempotencyRecord) error {
tenantAdp, err := a.infrastructureAdapter(id)
if err != nil {
return err
}
if record == nil || record.UserID.IsZero() {
return t.ErrMalformed
}
if tenantAdp != a {
return tenantAdp.TenantHTTPIdempotencyAbort(id, record)
}
_, err = a.db.Exec(`DELETE FROM im_http_idempotency
WHERE tenant_id=? AND user_id=? AND method=? AND route=? AND idempotency_key=? AND body_hash=? AND state=?`,
id, decodeUID(record.UserID), record.Method, record.Route, record.Key, record.BodyHash, t.HTTPIdempotencyProcessing)
return err
}
func (a *adapter) httpIdempotencyGet(tenantID t.TenantID, uid t.Uid, method, route, key string) (*t.HTTPIdempotencyRecord, error) {
row := struct {
UserID int64 `db:"user_id"`
Method string
Route string
Key string `db:"idempotency_key"`
BodyHash string `db:"body_hash"`
State string
ResponseStatus int `db:"response_status"`
ResponseBody []byte `db:"response_body"`
CreatedAt time.Time `db:"created_at"`
UpdatedAt time.Time `db:"updated_at"`
ExpiresAt time.Time `db:"expires_at"`
}{}
err := a.db.Get(&row, `SELECT user_id,method,route,idempotency_key,body_hash,state,response_status,response_body,created_at,updated_at,expires_at
FROM im_http_idempotency WHERE tenant_id=? AND user_id=? AND method=? AND route=? AND idempotency_key=?`,
tenantID, decodeUID(uid), method, route, key)
if err == sql.ErrNoRows {
return nil, t.ErrNotFound
}
if err != nil {
return nil, err
}
return &t.HTTPIdempotencyRecord{UserID: store.EncodeUid(row.UserID), Method: row.Method, Route: row.Route,
Key: row.Key, BodyHash: row.BodyHash, State: row.State, ResponseStatus: row.ResponseStatus,
ResponseBody: row.ResponseBody, CreatedAt: row.CreatedAt, UpdatedAt: row.UpdatedAt, ExpiresAt: row.ExpiresAt}, nil
}
func (a *adapter) TenantKafkaConsumeBegin(id t.TenantID, record *t.KafkaConsumeRecord, deadline time.Time) (*t.KafkaConsumeRecord, bool, error) {
tenantAdp, err := a.infrastructureAdapter(id)
if err != nil {
return nil, false, err
}
if record == nil || record.EventID == "" || record.ConsumerGroup == "" || record.SourceTopic == "" {
return nil, false, t.ErrMalformed
}
if tenantAdp != a {
return tenantAdp.TenantKafkaConsumeBegin(id, record, deadline)
}
now := t.TimeNow()
_, err = a.db.Exec(`INSERT INTO im_kafka_consume_log
(event_id,consumer_group,source_topic,event_type,biz_type,biz_id,status,processing_deadline,created_at,updated_at)
VALUES(?,?,?,?,?,?,?,?,?,?)`, record.EventID, record.ConsumerGroup, record.SourceTopic, record.EventType,
record.BizType, record.BizID, t.KafkaConsumeProcessing, deadline, now, now)
if err == nil {
record.Status = t.KafkaConsumeProcessing
record.ProcessingDeadline = &deadline
record.CreatedAt = now
record.UpdatedAt = now
return record, true, nil
}
if !isDupe(err) {
return nil, false, err
}
existing, err := a.kafkaConsumeGet(record.EventID, record.ConsumerGroup)
if err != nil {
return nil, false, err
}
if existing.Status == t.KafkaConsumeSucceeded || existing.Status == t.KafkaConsumeDead {
return existing, false, nil
}
if existing.Status == t.KafkaConsumeProcessing && existing.ProcessingDeadline != nil && existing.ProcessingDeadline.After(now) {
return existing, false, nil
}
res, err := a.db.Exec(`UPDATE im_kafka_consume_log SET status=?,source_topic=?,event_type=?,biz_type=?,biz_id=?,
processing_deadline=?,updated_at=? WHERE event_id=? AND consumer_group=? AND status<>?`,
t.KafkaConsumeProcessing, record.SourceTopic, record.EventType, record.BizType, record.BizID,
deadline, now, record.EventID, record.ConsumerGroup, t.KafkaConsumeSucceeded)
if err != nil {
return nil, false, err
}
if affected, _ := res.RowsAffected(); affected == 0 {
current, getErr := a.kafkaConsumeGet(record.EventID, record.ConsumerGroup)
return current, false, getErr
}
current, err := a.kafkaConsumeGet(record.EventID, record.ConsumerGroup)
return current, err == nil, err
}
func (a *adapter) TenantKafkaConsumeSucceed(id t.TenantID, eventID, consumerGroup string, now time.Time) error {
tenantAdp, err := a.infrastructureAdapter(id)
if err != nil {
return err
}
if eventID == "" || consumerGroup == "" {
return t.ErrMalformed
}
if tenantAdp != a {
return tenantAdp.TenantKafkaConsumeSucceed(id, eventID, consumerGroup, now)
}
res, err := a.db.Exec(`UPDATE im_kafka_consume_log SET status=?,succeeded_at=?,processing_deadline=NULL,updated_at=?
WHERE event_id=? AND consumer_group=?`, t.KafkaConsumeSucceeded, now, now, eventID, consumerGroup)
if err != nil {
return err
}
if affected, _ := res.RowsAffected(); affected == 0 {
return t.ErrNotFound
}
return nil
}
func (a *adapter) TenantKafkaConsumeFail(id t.TenantID, eventID, consumerGroup, reason, message string, now time.Time) (*t.KafkaConsumeRecord, error) {
tenantAdp, err := a.infrastructureAdapter(id)
if err != nil {
return nil, err
}
if eventID == "" || consumerGroup == "" {
return nil, t.ErrMalformed
}
if tenantAdp != a {
return tenantAdp.TenantKafkaConsumeFail(id, eventID, consumerGroup, reason, message, now)
}
_, err = a.db.Exec(`UPDATE im_kafka_consume_log SET status=?,retry_count=retry_count+1,last_reason=?,
last_error=?,processing_deadline=NULL,updated_at=? WHERE event_id=? AND consumer_group=?`,
t.KafkaConsumeFailed, truncateKafkaString(reason, 64), truncateKafkaString(message, 1000), now, eventID, consumerGroup)
if err != nil {
return nil, err
}
return a.kafkaConsumeGet(eventID, consumerGroup)
}
func (a *adapter) TenantKafkaConsumeDead(id t.TenantID, dead *t.KafkaDeadLetter, now time.Time) error {
tenantAdp, err := a.infrastructureAdapter(id)
if err != nil {
@@ -245,12 +34,7 @@ func (a *adapter) TenantKafkaConsumeDead(id t.TenantID, dead *t.KafkaDeadLetter,
if tenantAdp != a {
return tenantAdp.TenantKafkaConsumeDead(id, dead, now)
}
tx, err := a.db.BeginTxx(context.Background(), nil)
if err != nil {
return err
}
defer tx.Rollback()
if _, err = tx.Exec(`INSERT INTO im_kafka_dead_letters
if _, err = a.db.Exec(`INSERT INTO im_kafka_dead_letters
(dead_id,event_id,parent_event_id,source_topic,consumer_group,event_type,biz_type,biz_id,envelope,
retry_count,last_reason,last_error,status,replay_count,created_at,updated_at)
VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)
@@ -260,13 +44,7 @@ func (a *adapter) TenantKafkaConsumeDead(id t.TenantID, dead *t.KafkaDeadLetter,
truncateKafkaString(dead.LastError, 1000), t.KafkaDeadStatusDead, dead.ReplayCount, now, now); err != nil {
return err
}
if _, err = tx.Exec(`UPDATE im_kafka_consume_log SET status=?,dead_at=?,processing_deadline=NULL,last_reason=?,
last_error=?,updated_at=? WHERE event_id=? AND consumer_group=?`,
t.KafkaConsumeDead, now, truncateKafkaString(dead.LastReason, 64), truncateKafkaString(dead.LastError, 1000),
now, dead.EventID, dead.ConsumerGroup); err != nil {
return err
}
return tx.Commit()
return nil
}
func (a *adapter) TenantKafkaDeadLetterList(id t.TenantID, opts t.KafkaDeadLetterListOptions) (*t.KafkaDeadLetterPage, error) {
@@ -390,22 +168,7 @@ func (a *adapter) TenantKafkaDeadLetterMarkReplay(id t.TenantID, deadID, resolve
if tenantAdp != a {
return tenantAdp.TenantKafkaDeadLetterMarkReplay(id, deadID, resolvedBy, note, now)
}
tx, err := a.db.BeginTxx(context.Background(), nil)
if err != nil {
return nil, err
}
defer tx.Rollback()
var row struct {
EventID string `db:"event_id"`
ConsumerGroup string `db:"consumer_group"`
}
if err = tx.Get(&row, `SELECT event_id,consumer_group FROM im_kafka_dead_letters WHERE dead_id=? FOR UPDATE`, deadID); err != nil {
if err == sql.ErrNoRows {
return nil, t.ErrNotFound
}
return nil, err
}
res, err := tx.Exec(`UPDATE im_kafka_dead_letters SET status=?,replay_count=replay_count+1,resolved_by=?,
res, err := a.db.Exec(`UPDATE im_kafka_dead_letters SET status=?,replay_count=replay_count+1,resolved_by=?,
resolved_at=?,resolve_note=?,updated_at=? WHERE dead_id=? AND status IN (?,?)`,
t.KafkaDeadStatusPendingReplay, truncateKafkaString(resolvedBy, 64), now, truncateKafkaString(note, 1000),
now, deadID, t.KafkaDeadStatusDead, t.KafkaDeadStatusPendingReplay)
@@ -415,13 +178,6 @@ func (a *adapter) TenantKafkaDeadLetterMarkReplay(id t.TenantID, deadID, resolve
if affected, _ := res.RowsAffected(); affected == 0 {
return nil, t.ErrNotFound
}
if _, err = tx.Exec(`DELETE FROM im_kafka_consume_log WHERE event_id=? AND consumer_group=?`,
row.EventID, row.ConsumerGroup); err != nil {
return nil, err
}
if err = tx.Commit(); err != nil {
return nil, err
}
return a.TenantKafkaDeadLetterGet(id, deadID)
}
@@ -469,39 +225,6 @@ func scanKafkaDeadLetter(rows *sqlx.Rows) (*t.KafkaDeadLetter, error) {
}, nil
}
func (a *adapter) kafkaConsumeGet(eventID, consumerGroup string) (*t.KafkaConsumeRecord, error) {
row := struct {
EventID string `db:"event_id"`
ConsumerGroup string `db:"consumer_group"`
SourceTopic string `db:"source_topic"`
EventType string `db:"event_type"`
BizType string `db:"biz_type"`
BizID string `db:"biz_id"`
Status string `db:"status"`
RetryCount int `db:"retry_count"`
LastReason string `db:"last_reason"`
LastError string `db:"last_error"`
ProcessingDeadline *time.Time `db:"processing_deadline"`
SucceededAt *time.Time `db:"succeeded_at"`
DeadAt *time.Time `db:"dead_at"`
CreatedAt time.Time `db:"created_at"`
UpdatedAt time.Time `db:"updated_at"`
}{}
err := a.db.Get(&row, `SELECT event_id,consumer_group,source_topic,event_type,biz_type,biz_id,status,retry_count,
last_reason,last_error,processing_deadline,succeeded_at,dead_at,created_at,updated_at
FROM im_kafka_consume_log WHERE event_id=? AND consumer_group=?`, eventID, consumerGroup)
if err == sql.ErrNoRows {
return nil, t.ErrNotFound
}
if err != nil {
return nil, err
}
return &t.KafkaConsumeRecord{EventID: row.EventID, ConsumerGroup: row.ConsumerGroup, SourceTopic: row.SourceTopic,
EventType: row.EventType, BizType: row.BizType, BizID: row.BizID, Status: row.Status, RetryCount: row.RetryCount,
LastReason: row.LastReason, LastError: row.LastError, ProcessingDeadline: row.ProcessingDeadline,
SucceededAt: row.SucceededAt, DeadAt: row.DeadAt, CreatedAt: row.CreatedAt, UpdatedAt: row.UpdatedAt}, nil
}
func truncateKafkaString(value string, limit int) string {
if limit <= 0 {
return ""
@@ -519,10 +242,10 @@ func insertSystemNotificationTx(tx *sqlx.Tx, tenantID t.TenantID, uid t.Uid, eve
return err
}
now := t.TimeNow()
_, err = tx.Exec(`INSERT INTO im_system_notifications
_, err = tx.Exec(tenantShardSQL(tenantID, `INSERT INTO im_system_notifications
(tenant_id,user_id,event_id,event_type,title,summary,action,created_at,updated_at)
VALUES(?,?,?,?,?,?,?,?,?) ON DUPLICATE KEY UPDATE event_type=VALUES(event_type),title=VALUES(title),
summary=VALUES(summary),action=VALUES(action),updated_at=VALUES(updated_at)`, tenantID, decodeUID(uid), eventID,
summary=VALUES(summary),action=VALUES(action),updated_at=VALUES(updated_at)`), tenantID, decodeUID(uid), eventID,
eventType, title, summary, encoded, now, now)
return err
}