feat: shard group messages by month

2026-09-15 23:53:51 +08:00
parent b7c2536527
commit 139681fadb
25 changed files with 1718 additions and 247 deletions
@@ -0,0 +1,15 @@
+-- 在每个租户业务库执行。用于并发安全地生成 im_users.account_no。
+CREATE TABLE IF NOT EXISTS im_user_account_sequences (
+    id TINYINT UNSIGNED NOT NULL,
+    next_value INT UNSIGNED NOT NULL,
+    updated_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3)
+        ON UPDATE CURRENT_TIMESTAMP(3),
+    PRIMARY KEY (id)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
+
+-- 兼容已有用户数据，从当前最大账号编号继续递增，账号编号最小为 1000。
+INSERT INTO im_user_account_sequences (id, next_value)
+SELECT 1, GREATEST(COALESCE(MAX(account_no), 999), 999)
+FROM im_users
+ON DUPLICATE KEY UPDATE
+    next_value = GREATEST(next_value, VALUES(next_value));
@@ -1,3 +1,6 @@
+-- Legacy single-table baseline. New deployments must also run
+-- IM-群消息按月分表SQL.sql; this table remains only for migration-time dual reads.
+
 CREATE TABLE IF NOT EXISTS im_group_messages (
  id BIGINT NOT NULL AUTO_INCREMENT,
  topic VARCHAR(64) NOT NULL COMMENT '群 Topic，如 grp_xxx',
@@ -0,0 +1,107 @@
+-- IM group-message monthly sharding, baseline month: 2026-09 (UTC).
+-- Run in each tenant database. The application pre-creates subsequent months.
+
+CREATE TABLE IF NOT EXISTS im_group_message_shards (
+  shard_month CHAR(6) NOT NULL,
+  table_name VARCHAR(64) NOT NULL,
+  shard_start DATETIME(3) NOT NULL,
+  shard_end DATETIME(3) NOT NULL,
+  status VARCHAR(16) NOT NULL DEFAULT 'prepared',
+  row_count BIGINT NOT NULL DEFAULT 0,
+  min_created_at DATETIME(3) NULL,
+  max_created_at DATETIME(3) NULL,
+  created_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3),
+  updated_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3) ON UPDATE CURRENT_TIMESTAMP(3),
+  PRIMARY KEY (shard_month),
+  UNIQUE KEY uk_im_group_message_shards_table (table_name),
+  KEY idx_im_group_message_shards_status (status, shard_month)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
+
+CREATE TABLE IF NOT EXISTS im_group_message_topic_shards (
+  topic VARCHAR(64) NOT NULL,
+  shard_month CHAR(6) NOT NULL,
+  min_seq BIGINT NOT NULL,
+  max_seq BIGINT NOT NULL,
+  message_count BIGINT NOT NULL DEFAULT 0,
+  min_created_at DATETIME(3) NOT NULL,
+  max_created_at DATETIME(3) NOT NULL,
+  created_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3),
+  updated_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3) ON UPDATE CURRENT_TIMESTAMP(3),
+  PRIMARY KEY (topic, shard_month),
+  KEY idx_im_group_message_topic_shards_seq (topic, max_seq),
+  KEY idx_im_group_message_topic_shards_month (shard_month, topic)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
+
+CREATE TABLE IF NOT EXISTS im_group_message_dedup (
+  topic VARCHAR(64) NOT NULL,
+  from_user BIGINT NOT NULL,
+  client_msg_id VARCHAR(64) NOT NULL,
+  seqid BIGINT NOT NULL,
+  shard_month CHAR(6) NOT NULL,
+  message_id BIGINT NULL,
+  status VARCHAR(16) NOT NULL DEFAULT 'pending',
+  expires_at DATETIME(3) NOT NULL,
+  created_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3),
+  updated_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3) ON UPDATE CURRENT_TIMESTAMP(3),
+  PRIMARY KEY (topic, from_user, client_msg_id),
+  KEY idx_im_group_message_dedup_expires (expires_at),
+  KEY idx_im_group_message_dedup_topic_seq (topic, seqid)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
+
+CREATE TABLE IF NOT EXISTS im_group_messages_202609 (
+  message_id BIGINT NOT NULL,
+  topic VARCHAR(64) NOT NULL,
+  group_id BIGINT NOT NULL,
+  seqid BIGINT NOT NULL,
+  from_user BIGINT NOT NULL,
+  client_msg_id VARCHAR(64) NULL,
+  msg_type VARCHAR(32) NOT NULL DEFAULT 'text',
+  status VARCHAR(32) NOT NULL DEFAULT 'normal',
+  content JSON NULL,
+  head JSON NULL,
+  mentioned_user_ids JSON NULL,
+  mention_all TINYINT(1) NOT NULL DEFAULT 0,
+  reply_seqid BIGINT NULL,
+  root_seqid BIGINT NULL,
+  file_count INT NOT NULL DEFAULT 0,
+  content_size INT NOT NULL DEFAULT 0,
+  has_risk TINYINT(1) NOT NULL DEFAULT 0,
+  audit_status VARCHAR(32) NOT NULL DEFAULT 'pass',
+  version INT NOT NULL DEFAULT 1,
+  recalled_at DATETIME(3) NULL,
+  recalled_by BIGINT NULL,
+  deleted_at DATETIME(3) NULL,
+  expires_at DATETIME(3) NULL,
+  created_at DATETIME(3) NOT NULL,
+  updated_at DATETIME(3) NOT NULL,
+  PRIMARY KEY (message_id),
+  UNIQUE KEY uk_202609_group_seq (group_id, seqid),
+  UNIQUE KEY uk_202609_topic_seq (topic, seqid),
+  UNIQUE KEY uk_202609_sender_client (topic, from_user, client_msg_id),
+  KEY idx_202609_group_status_seq (group_id, status, seqid),
+  KEY idx_202609_topic_created (topic, created_at),
+  KEY idx_202609_topic_from_seq (topic, from_user, seqid),
+  KEY idx_202609_audit_created (audit_status, created_at),
+  KEY idx_202609_expires (expires_at)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
+
+CREATE TABLE IF NOT EXISTS im_group_messages_202608 LIKE im_group_messages_202609;
+CREATE TABLE IF NOT EXISTS im_group_messages_202610 LIKE im_group_messages_202609;
+CREATE TABLE IF NOT EXISTS im_group_messages_202611 LIKE im_group_messages_202609;
+
+INSERT INTO im_group_message_shards
+  (shard_month,table_name,shard_start,shard_end,status)
+VALUES
+  ('202608','im_group_messages_202608','2026-08-01 00:00:00.000','2026-09-01 00:00:00.000','sealing'),
+  ('202609','im_group_messages_202609','2026-09-01 00:00:00.000','2026-10-01 00:00:00.000','writable'),
+  ('202610','im_group_messages_202610','2026-10-01 00:00:00.000','2026-11-01 00:00:00.000','prepared'),
+  ('202611','im_group_messages_202611','2026-11-01 00:00:00.000','2026-12-01 00:00:00.000','prepared')
+ON DUPLICATE KEY UPDATE
+  table_name=VALUES(table_name),
+  shard_start=VALUES(shard_start),
+  shard_end=VALUES(shard_end),
+  status=IF(status IN ('sealed','archived'),status,VALUES(status));
+
+-- Historical migration is deliberately separate from schema deployment.
+-- Backfill im_group_messages into the matching month table in bounded batches,
+-- then populate im_group_message_topic_shards before removing legacy reads.