pro/im_server
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: complete shared database tenant sharding

Browse Source
This commit is contained in:
richjohnrambo
2026-09-23 02:19:07 +08:00
parent 1ec2feee47
commit e37b52c3c0
37 changed files with 946 additions and 1420 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/sql/IM-Kafka基础表SQL.sql
+3 -25
View File
@@ -2,32 +2,10 @@
-- Date: 2026-08-26
-- Scope: run once in every tenant business database, such as tinode.
-- Notes:
-- * Do not run in hotgo. Admin/HotGo side needs its own admin_command/outbox tables.
-- * Tables do not contain tenant_id because each tenant has an independent database.
-- * Do not run in hotgo. Admin/HotGo side needs its own admin_command tables.
-- * Kafka consume idempotency/retry state is Redis-backed; MySQL only keeps manual dead letters.
CREATE TABLE IF NOT EXISTS im_kafka_consume_log (
id BIGINT NOT NULL AUTO_INCREMENT,
event_id VARCHAR(96) NOT NULL,
consumer_group VARCHAR(128) NOT NULL,
source_topic VARCHAR(128) NOT NULL,
event_type VARCHAR(96) NOT NULL DEFAULT '',
biz_type VARCHAR(64) NOT NULL DEFAULT '',
biz_id VARCHAR(128) NOT NULL DEFAULT '',
status VARCHAR(16) NOT NULL DEFAULT 'processing',
retry_count INT NOT NULL DEFAULT 0,
last_reason VARCHAR(64) NOT NULL DEFAULT '',
last_error VARCHAR(1000) NOT NULL DEFAULT '',
processing_deadline DATETIME(3),
succeeded_at DATETIME(3),
dead_at DATETIME(3),
created_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3),
updated_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3) ON UPDATE CURRENT_TIMESTAMP(3),
PRIMARY KEY(id),
UNIQUE INDEX uk_im_kafka_consume_log_event_group(event_id, consumer_group),
INDEX idx_im_kafka_consume_log_status(status, processing_deadline),
INDEX idx_im_kafka_consume_log_biz(biz_type, biz_id),
INDEX idx_im_kafka_consume_log_topic(source_topic, id)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
DROP TABLE IF EXISTS im_kafka_consume_log;
CREATE TABLE IF NOT EXISTS im_kafka_dead_letters (
id BIGINT NOT NULL AUTO_INCREMENT,
docs/sql/IM-共库核心分表-仅本地.sql
+3 -23
View File
@@ -44,19 +44,9 @@ ALTER TABLE `im_group_message_shards`
ADD UNIQUE INDEX `uk_im_group_message_shards_table` (`tenant_id`, `table_name`),
ADD INDEX `idx_im_group_message_shards_status` (`tenant_id`, `status`, `shard_month`);
ALTER TABLE `im_group_message_topic_shards`
DROP PRIMARY KEY,
DROP INDEX `idx_im_group_message_topic_shards_seq`,
DROP INDEX `idx_im_group_message_topic_shards_month`,
ADD PRIMARY KEY (`tenant_id`, `topic`, `shard_month`),
ADD INDEX `idx_im_group_message_topic_shards_seq` (`tenant_id`, `topic`, `max_seq`),
ADD INDEX `idx_im_group_message_topic_shards_month` (`tenant_id`, `shard_month`, `topic`);
DROP TABLE IF EXISTS `im_group_message_topic_shards`;
ALTER TABLE `im_group_message_dedup`
DROP PRIMARY KEY,
DROP INDEX `idx_im_group_message_dedup_topic_seq`,
ADD PRIMARY KEY (`tenant_id`, `topic`, `from_user`, `client_msg_id`),
ADD INDEX `idx_im_group_message_dedup_topic_seq` (`tenant_id`, `topic`, `seqid`);
DROP TABLE IF EXISTS `im_group_message_dedup`;
-- 3. 创建好友与单聊消息分表并迁移现有数据。
CREATE TABLE IF NOT EXISTS `im_001_friendships` LIKE `im_friendships`;
@@ -146,17 +136,7 @@ SELECT 2,`shard_month`,CONCAT('im_002_group_messages_', `shard_month`),`shard_st
FROM `meeting_im`.`im_group_message_shards`
ON DUPLICATE KEY UPDATE `table_name`=VALUES(`table_name`),`status`=VALUES(`status`);
INSERT IGNORE INTO `im_group_message_topic_shards` SELECT * FROM `meeting_im`.`im_group_message_topic_shards`;
INSERT IGNORE INTO `im_group_message_dedup` SELECT * FROM `meeting_im`.`im_group_message_dedup`;
-- 旧非月表迁移后补齐 Topic 路由，并按物理表实际数据校准分片统计。
INSERT INTO `im_group_message_topic_shards`
(`tenant_id`,`topic`,`shard_month`,`min_seq`,`max_seq`,`message_count`,`min_created_at`,`max_created_at`)
SELECT 1,`topic`,'202608',MIN(`seqid`),MAX(`seqid`),COUNT(*),MIN(`created_at`),MAX(`created_at`)
FROM `im_001_group_messages_202608`
GROUP BY `topic`
ON DUPLICATE KEY UPDATE `min_seq`=VALUES(`min_seq`),`max_seq`=VALUES(`max_seq`),
`message_count`=VALUES(`message_count`),`min_created_at`=VALUES(`min_created_at`),`max_created_at`=VALUES(`max_created_at`);
-- Topic-to-month routes are Redis-backed and are rebuilt by runtime reads.
UPDATE `im_group_message_shards` AS s
SET s.`row_count`=(SELECT COUNT(*) FROM `im_001_group_messages_202608`),
docs/sql/IM-共库核心分表-测试预演库.sql
+3 -27
View File
@@ -59,19 +59,9 @@ ALTER TABLE `im_group_message_shards`
ADD UNIQUE INDEX `uk_im_group_message_shards_table` (`tenant_id`,`table_name`),
ADD INDEX `idx_im_group_message_shards_status` (`tenant_id`,`status`,`shard_month`);
ALTER TABLE `im_group_message_topic_shards`
DROP PRIMARY KEY,
DROP INDEX `idx_im_group_message_topic_shards_seq`,
DROP INDEX `idx_im_group_message_topic_shards_month`,
ADD PRIMARY KEY (`tenant_id`,`topic`,`shard_month`),
ADD INDEX `idx_im_group_message_topic_shards_seq` (`tenant_id`,`topic`,`max_seq`),
ADD INDEX `idx_im_group_message_topic_shards_month` (`tenant_id`,`shard_month`,`topic`);
DROP TABLE IF EXISTS `im_group_message_topic_shards`;
ALTER TABLE `im_group_message_dedup`
DROP PRIMARY KEY,
DROP INDEX `idx_im_group_message_dedup_topic_seq`,
ADD PRIMARY KEY (`tenant_id`,`topic`,`from_user`,`client_msg_id`),
ADD INDEX `idx_im_group_message_dedup_topic_seq` (`tenant_id`,`topic`,`seqid`);
DROP TABLE IF EXISTS `im_group_message_dedup`;
-- 3. 创建商户 2 的好友与单聊消息分表并复制快照数据。
CREATE TABLE `im_002_friendships` LIKE `im_friendships`;
@@ -208,21 +198,7 @@ SELECT 1,`shard_month`,CONCAT('im_001_group_messages_',`shard_month`),`shard_sta
`status`,`row_count`,`min_created_at`,`max_created_at`,`created_at`,`updated_at`
FROM `tinode`.`im_group_message_shards`;
INSERT INTO `im_group_message_topic_shards` (
`tenant_id`,`topic`,`shard_month`,`min_seq`,`max_seq`,`message_count`,
`min_created_at`,`max_created_at`,`created_at`,`updated_at`
)
SELECT 1,`topic`,`shard_month`,`min_seq`,`max_seq`,`message_count`,
`min_created_at`,`max_created_at`,`created_at`,`updated_at`
FROM `tinode`.`im_group_message_topic_shards`;
INSERT INTO `im_group_message_dedup` (
`tenant_id`,`topic`,`from_user`,`client_msg_id`,`seqid`,`shard_month`,
`message_id`,`status`,`expires_at`,`created_at`,`updated_at`
)
SELECT 1,`topic`,`from_user`,`client_msg_id`,`seqid`,`shard_month`,
`message_id`,`status`,`expires_at`,`created_at`,`updated_at`
FROM `tinode`.`im_group_message_dedup`;
-- Topic-to-month routes are Redis-backed and are rebuilt by runtime reads.
-- 以实际复制后的物理表为准刷新统计，避免源库路由统计滞后。
DELIMITER $$
docs/sql/IM-共库运行态缺口修复SQL.sql
+12 -45
View File
@@ -29,52 +29,19 @@ SELECT id, 1, 999
FROM hotgo.im_tenant
WHERE state = 1;
CREATE TABLE IF NOT EXISTS im_login_lockouts (
id BIGINT NOT NULL,
user_id BIGINT NULL,
account_hash CHAR(64) NOT NULL,
failed_attempts INT NOT NULL DEFAULT 0,
window_started_at DATETIME(3) NULL,
locked_until DATETIME(3) NULL,
last_attempt_at DATETIME(3) NULL,
created_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3),
updated_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3)
ON UPDATE CURRENT_TIMESTAMP(3),
PRIMARY KEY (id),
UNIQUE INDEX uk_im_login_lockouts_account (account_hash),
INDEX idx_im_login_lockouts_user (user_id),
INDEX idx_im_login_lockouts_locked (locked_until)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
DROP TABLE IF EXISTS im_login_lockouts;
-- HTTP write idempotency must be tenant scoped in the shared meeting_im DB.
CREATE TABLE IF NOT EXISTS im_http_idempotency (
id BIGINT NOT NULL AUTO_INCREMENT,
tenant_id BIGINT NOT NULL DEFAULT 0,
user_id BIGINT NOT NULL,
method VARCHAR(8) NOT NULL,
route VARCHAR(512) NOT NULL,
idempotency_key VARCHAR(128) NOT NULL,
body_hash CHAR(64) NOT NULL,
state VARCHAR(16) NOT NULL DEFAULT 'processing',
response_status INT NOT NULL DEFAULT 0,
response_body MEDIUMBLOB,
expires_at DATETIME(3) NOT NULL,
created_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3),
updated_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3)
ON UPDATE CURRENT_TIMESTAMP(3),
PRIMARY KEY (id),
UNIQUE INDEX uk_im_http_idempotency_scope
(tenant_id, user_id, method, route, idempotency_key),
INDEX idx_im_http_idempotency_expiry (expires_at)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
ALTER TABLE im_http_idempotency
ADD COLUMN tenant_id BIGINT NOT NULL DEFAULT 0 AFTER id;
ALTER TABLE im_http_idempotency
DROP INDEX uk_im_http_idempotency_scope;
ALTER TABLE im_http_idempotency
ADD UNIQUE INDEX uk_im_http_idempotency_scope
(tenant_id, user_id, method, route, idempotency_key);
-- HTTP write idempotency is Redis-backed. Keep it out of MySQL.
DROP TABLE IF EXISTS im_http_idempotency;
DROP TABLE IF EXISTS im_group_idempotency;
DROP TABLE IF EXISTS im_group_message_dedup;
DROP TABLE IF EXISTS im_group_message_topic_shards;
DROP TABLE IF EXISTS im_kafka_outbox;
DROP TABLE IF EXISTS im_kafka_consume_log;
DROP TABLE IF EXISTS im_business_events_outbox;
DROP TABLE IF EXISTS im_group_events_outbox;
DROP TABLE IF EXISTS im_login_failure_logs;
DROP TABLE IF EXISTS im_action_tokens;
-- User identity indexes must be unique inside one tenant, not globally.
ALTER TABLE im_users
docs/sql/IM-后台消息与登录审计SQL.sql
+2 -43
View File
@@ -30,50 +30,9 @@ CREATE TABLE IF NOT EXISTS im_user_login_logs (
INDEX idx_im_user_login_request (request_id)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
CREATE TABLE IF NOT EXISTS im_login_failure_logs (
id BIGINT NOT NULL AUTO_INCREMENT,
user_id BIGINT NULL,
account_type VARCHAR(24) NOT NULL DEFAULT '',
account_hash CHAR(64) NOT NULL DEFAULT '',
account_mask VARCHAR(128) NOT NULL DEFAULT '',
failure_type VARCHAR(32) NOT NULL,
reason VARCHAR(96) NOT NULL DEFAULT '',
detail VARCHAR(500) NOT NULL DEFAULT '',
ip VARCHAR(64) NOT NULL DEFAULT '',
region VARCHAR(128) NOT NULL DEFAULT '',
device_id VARCHAR(255) NOT NULL DEFAULT '',
platform VARCHAR(32) NOT NULL DEFAULT 'unknown',
app_version VARCHAR(32) NOT NULL DEFAULT '',
user_agent VARCHAR(512) NOT NULL DEFAULT '',
risk_level VARCHAR(16) NOT NULL DEFAULT 'medium',
risk_action VARCHAR(24) NOT NULL DEFAULT 'audit_only',
request_id VARCHAR(64) NOT NULL DEFAULT '',
attempt_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3),
metadata JSON NULL,
created_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3),
PRIMARY KEY (id),
INDEX idx_im_login_failure_account_time (account_hash, attempt_at DESC, id DESC),
INDEX idx_im_login_failure_user_time (user_id, attempt_at DESC, id DESC),
INDEX idx_im_login_failure_ip_time (ip, attempt_at DESC, id DESC),
INDEX idx_im_login_failure_type_time (failure_type, attempt_at DESC, id DESC),
INDEX idx_im_login_failure_request (request_id)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
DROP TABLE IF EXISTS im_login_failure_logs;
CREATE TABLE IF NOT EXISTS im_login_lockouts (
id BIGINT NOT NULL,
user_id BIGINT NULL,
account_hash CHAR(64) NOT NULL,
failed_attempts INT NOT NULL DEFAULT 0,
window_started_at DATETIME(3) NULL,
locked_until DATETIME(3) NULL,
last_attempt_at DATETIME(3) NULL,
created_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3),
updated_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3) ON UPDATE CURRENT_TIMESTAMP(3),
PRIMARY KEY (id),
UNIQUE INDEX uk_im_login_lockouts_account (account_hash),
INDEX idx_im_login_lockouts_user (user_id),
INDEX idx_im_login_lockouts_locked (locked_until)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
DROP TABLE IF EXISTS im_login_lockouts;
CREATE TABLE IF NOT EXISTS im_message_admin_actions (
id BIGINT NOT NULL AUTO_INCREMENT,
docs/sql/IM-群消息按月分表SQL.sql
+3 -30
View File
@@ -17,36 +17,9 @@ CREATE TABLE IF NOT EXISTS im_group_message_shards (
KEY idx_im_group_message_shards_status (status, shard_month)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
CREATE TABLE IF NOT EXISTS im_group_message_topic_shards (
topic VARCHAR(64) NOT NULL,
shard_month CHAR(6) NOT NULL,
min_seq BIGINT NOT NULL,
max_seq BIGINT NOT NULL,
message_count BIGINT NOT NULL DEFAULT 0,
min_created_at DATETIME(3) NOT NULL,
max_created_at DATETIME(3) NOT NULL,
created_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3),
updated_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3) ON UPDATE CURRENT_TIMESTAMP(3),
PRIMARY KEY (topic, shard_month),
KEY idx_im_group_message_topic_shards_seq (topic, max_seq),
KEY idx_im_group_message_topic_shards_month (shard_month, topic)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
DROP TABLE IF EXISTS im_group_message_topic_shards;
CREATE TABLE IF NOT EXISTS im_group_message_dedup (
topic VARCHAR(64) NOT NULL,
from_user BIGINT NOT NULL,
client_msg_id VARCHAR(64) NOT NULL,
seqid BIGINT NOT NULL,
shard_month CHAR(6) NOT NULL,
message_id BIGINT NULL,
status VARCHAR(16) NOT NULL DEFAULT 'pending',
expires_at DATETIME(3) NOT NULL,
created_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3),
updated_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3) ON UPDATE CURRENT_TIMESTAMP(3),
PRIMARY KEY (topic, from_user, client_msg_id),
KEY idx_im_group_message_dedup_expires (expires_at),
KEY idx_im_group_message_dedup_topic_seq (topic, seqid)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
DROP TABLE IF EXISTS im_group_message_dedup;
CREATE TABLE IF NOT EXISTS im_group_messages_202609 (
message_id BIGINT NOT NULL,
@@ -104,4 +77,4 @@ ON DUPLICATE KEY UPDATE
-- Historical migration is deliberately separate from schema deployment.
-- Backfill im_group_messages into the matching month table in bounded batches,
-- then populate im_group_message_topic_shards before removing legacy reads.
-- then warm Redis topic routes before removing legacy reads.