docs(sql): add shared database sharding migrations
This commit is contained in:
richjohnrambo
@@ -0,0 +1,173 @@
|
||||
-- 仅用于本地环境：将高数据量表迁移为“共库、按商户分表”。
|
||||
-- 目标命名：
|
||||
-- im_001_friendships
|
||||
-- im_001_messages
|
||||
-- im_001_group_messages_YYYYMM
|
||||
-- 测试环境不得执行本文件。
|
||||
|
||||
-- 1. 商户编号改为事务序列分配，下一编号按当前最大 ID 顺序递增。
|
||||
USE `hotgo`;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS `im_tenant_id_sequence` (
|
||||
`sequence_name` VARCHAR(32) NOT NULL,
|
||||
`next_id` BIGINT NOT NULL,
|
||||
`updated_at` DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3) ON UPDATE CURRENT_TIMESTAMP(3),
|
||||
PRIMARY KEY (`sequence_name`)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
|
||||
|
||||
INSERT INTO `im_tenant_id_sequence` (`sequence_name`, `next_id`)
|
||||
SELECT 'tenant', COALESCE(MAX(`id`), 0) + 1 FROM `im_tenant`
|
||||
ON DUPLICATE KEY UPDATE `next_id` = GREATEST(`next_id`, VALUES(`next_id`));
|
||||
|
||||
-- 禁止绕过序列表使用 AUTO_INCREMENT 生成跳号或历史大整数编号。
|
||||
ALTER TABLE `hg_admin_member` DROP FOREIGN KEY `fk_hg_admin_member_im_tenant`;
|
||||
ALTER TABLE `im_tenant_config` DROP FOREIGN KEY `fk_im_tenant_config_tenant`;
|
||||
ALTER TABLE `im_tenant` MODIFY `id` BIGINT NOT NULL;
|
||||
ALTER TABLE `hg_admin_member` ADD CONSTRAINT `fk_hg_admin_member_im_tenant`
|
||||
FOREIGN KEY (`tenant_id`) REFERENCES `im_tenant` (`id`);
|
||||
ALTER TABLE `im_tenant_config` ADD CONSTRAINT `fk_im_tenant_config_tenant`
|
||||
FOREIGN KEY (`tenant_id`) REFERENCES `im_tenant` (`id`);
|
||||
|
||||
UPDATE `im_tenant` AS target
|
||||
JOIN `im_tenant` AS shared ON shared.`code`='default'
|
||||
SET target.`db_name`=shared.`db_name`, target.`db_dsn`=shared.`db_dsn`
|
||||
WHERE target.`id`<>shared.`id`;
|
||||
|
||||
-- 2. 月表路由元数据改为以 tenant_id 为第一维度。
|
||||
USE `tinode`;
|
||||
|
||||
ALTER TABLE `im_group_message_shards`
|
||||
DROP PRIMARY KEY,
|
||||
DROP INDEX `uk_im_group_message_shards_table`,
|
||||
DROP INDEX `idx_im_group_message_shards_status`,
|
||||
ADD PRIMARY KEY (`tenant_id`, `shard_month`),
|
||||
ADD UNIQUE INDEX `uk_im_group_message_shards_table` (`tenant_id`, `table_name`),
|
||||
ADD INDEX `idx_im_group_message_shards_status` (`tenant_id`, `status`, `shard_month`);
|
||||
|
||||
ALTER TABLE `im_group_message_topic_shards`
|
||||
DROP PRIMARY KEY,
|
||||
DROP INDEX `idx_im_group_message_topic_shards_seq`,
|
||||
DROP INDEX `idx_im_group_message_topic_shards_month`,
|
||||
ADD PRIMARY KEY (`tenant_id`, `topic`, `shard_month`),
|
||||
ADD INDEX `idx_im_group_message_topic_shards_seq` (`tenant_id`, `topic`, `max_seq`),
|
||||
ADD INDEX `idx_im_group_message_topic_shards_month` (`tenant_id`, `shard_month`, `topic`);
|
||||
|
||||
ALTER TABLE `im_group_message_dedup`
|
||||
DROP PRIMARY KEY,
|
||||
DROP INDEX `idx_im_group_message_dedup_topic_seq`,
|
||||
ADD PRIMARY KEY (`tenant_id`, `topic`, `from_user`, `client_msg_id`),
|
||||
ADD INDEX `idx_im_group_message_dedup_topic_seq` (`tenant_id`, `topic`, `seqid`);
|
||||
|
||||
-- 3. 创建好友与单聊消息分表并迁移现有数据。
|
||||
CREATE TABLE IF NOT EXISTS `im_001_friendships` LIKE `im_friendships`;
|
||||
ALTER TABLE `im_001_friendships` MODIFY `tenant_id` BIGINT NOT NULL DEFAULT 1 COMMENT '商户ID';
|
||||
INSERT IGNORE INTO `im_001_friendships` SELECT * FROM `im_friendships` WHERE `tenant_id` = 1;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS `im_001_messages` LIKE `im_messages`;
|
||||
ALTER TABLE `im_001_messages` MODIFY `tenant_id` BIGINT NOT NULL DEFAULT 1 COMMENT '商户ID';
|
||||
INSERT IGNORE INTO `im_001_messages` SELECT * FROM `im_messages` WHERE `tenant_id` = 1;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS `im_002_friendships` LIKE `im_friendships`;
|
||||
ALTER TABLE `im_002_friendships` MODIFY `tenant_id` BIGINT NOT NULL DEFAULT 2 COMMENT '商户ID';
|
||||
INSERT IGNORE INTO `im_002_friendships` SELECT * FROM `meeting_im`.`im_friendships`;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS `im_002_messages` LIKE `im_messages`;
|
||||
ALTER TABLE `im_002_messages` MODIFY `tenant_id` BIGINT NOT NULL DEFAULT 2 COMMENT '商户ID';
|
||||
INSERT IGNORE INTO `im_002_messages` SELECT * FROM `meeting_im`.`im_messages`;
|
||||
|
||||
-- 4. 迁移所有已存在的标准月表。过程只识别 im_group_messages_YYYYMM。
|
||||
DELIMITER $$
|
||||
|
||||
DROP PROCEDURE IF EXISTS migrate_group_month_tables$$
|
||||
CREATE PROCEDURE migrate_group_month_tables(IN source_schema VARCHAR(64), IN source_tenant BIGINT, IN tenant_code CHAR(3))
|
||||
BEGIN
|
||||
DECLARE done INT DEFAULT 0;
|
||||
DECLARE source_table VARCHAR(64);
|
||||
DECLARE month_code CHAR(6);
|
||||
DECLARE target_table VARCHAR(64);
|
||||
DECLARE month_cursor CURSOR FOR
|
||||
SELECT table_name
|
||||
FROM information_schema.tables
|
||||
WHERE table_schema = source_schema
|
||||
AND table_name REGEXP '^im_group_messages_[0-9]{6}$';
|
||||
DECLARE CONTINUE HANDLER FOR NOT FOUND SET done = 1;
|
||||
|
||||
OPEN month_cursor;
|
||||
month_loop: LOOP
|
||||
FETCH month_cursor INTO source_table;
|
||||
IF done = 1 THEN
|
||||
LEAVE month_loop;
|
||||
END IF;
|
||||
SET month_code = RIGHT(source_table, 6);
|
||||
SET target_table = CONCAT('im_', tenant_code, '_group_messages_', month_code);
|
||||
SET @create_sql = CONCAT('CREATE TABLE IF NOT EXISTS `tinode`.`', target_table,
|
||||
'` LIKE `', source_schema, '`.`', source_table, '`');
|
||||
PREPARE stmt FROM @create_sql; EXECUTE stmt; DEALLOCATE PREPARE stmt;
|
||||
SET @alter_sql = CONCAT('ALTER TABLE `tinode`.`', target_table,
|
||||
'` MODIFY `tenant_id` BIGINT NOT NULL DEFAULT ', source_tenant, ' COMMENT ''商户ID''');
|
||||
PREPARE stmt FROM @alter_sql; EXECUTE stmt; DEALLOCATE PREPARE stmt;
|
||||
SET @copy_sql = CONCAT('INSERT IGNORE INTO `tinode`.`', target_table,
|
||||
'` SELECT * FROM `', source_schema, '`.`', source_table,
|
||||
'` WHERE `tenant_id`=', source_tenant);
|
||||
PREPARE stmt FROM @copy_sql; EXECUTE stmt; DEALLOCATE PREPARE stmt;
|
||||
END LOOP;
|
||||
CLOSE month_cursor;
|
||||
END$$
|
||||
|
||||
DELIMITER ;
|
||||
|
||||
CALL migrate_group_month_tables('tinode', 1, '001');
|
||||
CALL migrate_group_month_tables('meeting_im', 2, '002');
|
||||
DROP PROCEDURE IF EXISTS migrate_group_month_tables;
|
||||
|
||||
-- 旧的非月表数据按创建月份归入 1 号商户月表（本地现有数据为 202608）。
|
||||
INSERT IGNORE INTO `im_001_group_messages_202608`
|
||||
(`tenant_id`,`message_id`,`topic`,`group_id`,`seqid`,`from_user`,`client_msg_id`,`msg_type`,`status`,
|
||||
`content`,`content_nonce`,`key_version`,`head`,`mentioned_user_ids`,`mention_all`,`reply_seqid`,`root_seqid`,
|
||||
`file_count`,`content_size`,`has_risk`,`audit_status`,`version`,`recalled_at`,`recalled_by`,`deleted_at`,
|
||||
`expires_at`,`created_at`,`updated_at`)
|
||||
SELECT 1,`id`,`topic`,`group_id`,`seqid`,`from_user`,`client_msg_id`,`msg_type`,`status`,
|
||||
`content`,`content_nonce`,`key_version`,`head`,`mentioned_user_ids`,`mention_all`,`reply_seqid`,`root_seqid`,
|
||||
`file_count`,`content_size`,`has_risk`,`audit_status`,`version`,`recalled_at`,`recalled_by`,`deleted_at`,
|
||||
`expires_at`,`created_at`,`updated_at`
|
||||
FROM `im_group_messages`
|
||||
WHERE `tenant_id` = 1 AND DATE_FORMAT(`created_at`, '%Y%m') = '202608';
|
||||
|
||||
-- 5. 修正 1 号商户路由，并合入 2 号商户路由元数据。
|
||||
UPDATE `im_group_message_shards`
|
||||
SET `table_name` = CONCAT('im_001_group_messages_', `shard_month`)
|
||||
WHERE `tenant_id` = 1;
|
||||
|
||||
INSERT INTO `im_group_message_shards`
|
||||
(`tenant_id`,`shard_month`,`table_name`,`shard_start`,`shard_end`,`status`,`row_count`,
|
||||
`min_created_at`,`max_created_at`,`created_at`,`updated_at`)
|
||||
SELECT 2,`shard_month`,CONCAT('im_002_group_messages_', `shard_month`),`shard_start`,`shard_end`,`status`,
|
||||
`row_count`,`min_created_at`,`max_created_at`,`created_at`,`updated_at`
|
||||
FROM `meeting_im`.`im_group_message_shards`
|
||||
ON DUPLICATE KEY UPDATE `table_name`=VALUES(`table_name`),`status`=VALUES(`status`);
|
||||
|
||||
INSERT IGNORE INTO `im_group_message_topic_shards` SELECT * FROM `meeting_im`.`im_group_message_topic_shards`;
|
||||
INSERT IGNORE INTO `im_group_message_dedup` SELECT * FROM `meeting_im`.`im_group_message_dedup`;
|
||||
|
||||
-- 旧非月表迁移后补齐 Topic 路由，并按物理表实际数据校准分片统计。
|
||||
INSERT INTO `im_group_message_topic_shards`
|
||||
(`tenant_id`,`topic`,`shard_month`,`min_seq`,`max_seq`,`message_count`,`min_created_at`,`max_created_at`)
|
||||
SELECT 1,`topic`,'202608',MIN(`seqid`),MAX(`seqid`),COUNT(*),MIN(`created_at`),MAX(`created_at`)
|
||||
FROM `im_001_group_messages_202608`
|
||||
GROUP BY `topic`
|
||||
ON DUPLICATE KEY UPDATE `min_seq`=VALUES(`min_seq`),`max_seq`=VALUES(`max_seq`),
|
||||
`message_count`=VALUES(`message_count`),`min_created_at`=VALUES(`min_created_at`),`max_created_at`=VALUES(`max_created_at`);
|
||||
|
||||
UPDATE `im_group_message_shards` AS s
|
||||
SET s.`row_count`=(SELECT COUNT(*) FROM `im_001_group_messages_202608`),
|
||||
s.`min_created_at`=(SELECT MIN(`created_at`) FROM `im_001_group_messages_202608`),
|
||||
s.`max_created_at`=(SELECT MAX(`created_at`) FROM `im_001_group_messages_202608`)
|
||||
WHERE s.`tenant_id`=1 AND s.`shard_month`='202608';
|
||||
|
||||
-- 验证结果。
|
||||
SELECT `sequence_name`,`next_id` FROM `hotgo`.`im_tenant_id_sequence`;
|
||||
SELECT `table_name`,`table_rows`
|
||||
FROM information_schema.tables
|
||||
WHERE table_schema = 'tinode'
|
||||
AND table_name REGEXP '^im_[0-9]{3}_(friendships|messages|group_messages_[0-9]{6})$'
|
||||
ORDER BY table_name;
|
||||
Reference in New Issue
Block a user