pro/im_server
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: complete shared database tenant sharding

Browse Source
This commit is contained in:
richjohnrambo
2026-09-23 02:19:07 +08:00
parent 1ec2feee47
commit e37b52c3c0
37 changed files with 946 additions and 1420 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/db/mysql/group_message_shards.go
+18 -39
View File
@@ -25,24 +25,6 @@ func ensureGroupMessageShardInfrastructure(exec sqlExecutor, tenantID t.TenantID
PRIMARY KEY(tenant_id, shard_month), UNIQUE INDEX uk_im_group_message_shards_table(tenant_id, table_name),
INDEX idx_im_group_message_shards_status(tenant_id, status, shard_month)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci`,
`CREATE TABLE IF NOT EXISTS im_group_message_topic_shards (
tenant_id BIGINT NOT NULL DEFAULT 0, topic VARCHAR(64) NOT NULL, shard_month CHAR(6) NOT NULL,
min_seq BIGINT NOT NULL, max_seq BIGINT NOT NULL, message_count BIGINT NOT NULL DEFAULT 0,
min_created_at DATETIME(3) NOT NULL, max_created_at DATETIME(3) NOT NULL,
created_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3),
updated_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3) ON UPDATE CURRENT_TIMESTAMP(3),
PRIMARY KEY(tenant_id, topic, shard_month), INDEX idx_im_group_message_topic_shards_seq(tenant_id, topic, max_seq),
INDEX idx_im_group_message_topic_shards_month(tenant_id, shard_month, topic)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci`,
`CREATE TABLE IF NOT EXISTS im_group_message_dedup (
tenant_id BIGINT NOT NULL DEFAULT 0, topic VARCHAR(64) NOT NULL, from_user BIGINT NOT NULL, client_msg_id VARCHAR(64) NOT NULL,
seqid BIGINT NOT NULL, shard_month CHAR(6) NOT NULL, message_id BIGINT,
status VARCHAR(16) NOT NULL DEFAULT 'pending', expires_at DATETIME(3) NOT NULL,
created_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3),
updated_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3) ON UPDATE CURRENT_TIMESTAMP(3),
PRIMARY KEY(tenant_id, topic, from_user, client_msg_id), INDEX idx_im_group_message_dedup_expires(expires_at),
INDEX idx_im_group_message_dedup_topic_seq(tenant_id, topic, seqid)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci`,
}
for _, statement := range statements {
if _, err := exec.Exec(statement); err != nil {
@@ -129,22 +111,13 @@ func groupMessageShardTable(tenantID t.TenantID, month string) (string, bool) {
func groupMessageTablesForTopic(q sqlx.QueryerContext, ctx context.Context, tenantID t.TenantID, topic string) ([]string, error) {
routes, cached, err := store.GetGroupMessageShardRoutes(tenantID, topic)
redisAvailable := err == nil
if err != nil {
cached = false
}
if !cached {
routes = nil
if err = sqlx.SelectContext(ctx, q, &routes,
`SELECT shard_month AS month,min_seq,max_seq,message_count AS count
FROM im_group_message_topic_shards WHERE tenant_id=? AND topic=? ORDER BY shard_month DESC`, tenantID, topic); err != nil {
if routes, err = groupMessageAllShardRoutes(q, ctx, tenantID); err != nil {
return nil, err
}
if redisAvailable {
for _, route := range routes {
_ = store.PutGroupMessageShardRoute(tenantID, topic, route)
}
}
}
tables := make([]string, 0, len(routes)+1)
for _, route := range routes {
@@ -157,23 +130,13 @@ func groupMessageTablesForTopic(q sqlx.QueryerContext, ctx context.Context, tena
func groupMessageTablesForSeq(q sqlx.QueryerContext, ctx context.Context, tenantID t.TenantID, topic string, seq int64) ([]string, error) {
routes, cached, err := store.GetGroupMessageShardRoutes(tenantID, topic)
redisAvailable := err == nil
if err != nil {
cached = false
}
if !cached {
routes = nil
if err = sqlx.SelectContext(ctx, q, &routes,
`SELECT shard_month AS month,min_seq,max_seq,message_count AS count
FROM im_group_message_topic_shards WHERE tenant_id=? AND topic=? AND min_seq<=? AND max_seq>=?
ORDER BY shard_month DESC`, tenantID, topic, seq, seq); err != nil {
if routes, err = groupMessageAllShardRoutes(q, ctx, tenantID); err != nil {
return nil, err
}
if redisAvailable {
for _, route := range routes {
_ = store.PutGroupMessageShardRoute(tenantID, topic, route)
}
}
}
tables := make([]string, 0, 2)
for _, route := range routes {
@@ -186,6 +149,22 @@ func groupMessageTablesForSeq(q sqlx.QueryerContext, ctx context.Context, tenant
return tables, nil
}
func groupMessageAllShardRoutes(q sqlx.QueryerContext, ctx context.Context, tenantID t.TenantID) ([]store.GroupMessageShardRoute, error) {
var rows []struct {
Month string `db:"month"`
}
if err := sqlx.SelectContext(ctx, q, &rows, `SELECT shard_month AS month FROM im_group_message_shards WHERE tenant_id=? ORDER BY shard_month DESC`, tenantID); err != nil {
return nil, err
}
routes := make([]store.GroupMessageShardRoute, 0, len(rows))
for _, row := range rows {
if validShardMonth(row.Month) {
routes = append(routes, store.GroupMessageShardRoute{Month: row.Month, MinSeq: 1, MaxSeq: 1<<62 - 1, Count: 0})
}
}
return routes, nil
}
func groupMessageMonthFromTable(table string) string {
const suffixLength = 6
if len(table) > suffixLength && validShardMonth(table[len(table)-suffixLength:]) {