diff --git a/docs/IM-数据库变更汇总.sql b/docs/IM-数据库变更汇总.sql index 4059e68..e6a6c47 100644 --- a/docs/IM-数据库变更汇总.sql +++ b/docs/IM-数据库变更汇总.sql @@ -110,22 +110,7 @@ CREATE TABLE IF NOT EXISTS tinode.im_realname_verifications ( INDEX idx_im_realname_id_hash (id_number_hash) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; -CREATE TABLE IF NOT EXISTS tinode.im_action_tokens ( - id BIGINT NOT NULL AUTO_INCREMENT, - user_id BIGINT NOT NULL, - token_id VARCHAR(64) NOT NULL, - action VARCHAR(64) NOT NULL, - target VARCHAR(128) NOT NULL DEFAULT '', - token_hash CHAR(64) NOT NULL, - state SMALLINT NOT NULL, - expires_at DATETIME(3) NOT NULL, - consumed_at DATETIME(3), - created_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3), - updated_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3) ON UPDATE CURRENT_TIMESTAMP(3), - PRIMARY KEY (id), - UNIQUE INDEX uk_im_action_tokens_token (token_id), - INDEX idx_im_action_tokens_user_action (user_id, action, target, state, expires_at) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; +DROP TABLE IF EXISTS tinode.im_action_tokens; CREATE TABLE IF NOT EXISTS tinode.im_youth_mode_settings ( id BIGINT NOT NULL, @@ -164,24 +149,9 @@ CREATE TABLE IF NOT EXISTS tinode.im_system_notifications ( INDEX idx_im_system_notifications_inbox (user_id, id DESC) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; -CREATE TABLE IF NOT EXISTS tinode.im_http_idempotency ( - id BIGINT NOT NULL AUTO_INCREMENT, - tenant_id BIGINT NOT NULL DEFAULT 0, - user_id BIGINT NOT NULL, - method VARCHAR(8) NOT NULL, - route VARCHAR(512) NOT NULL, - idempotency_key VARCHAR(128) NOT NULL, - body_hash CHAR(64) NOT NULL, - state VARCHAR(16) NOT NULL DEFAULT 'processing', - response_status INT NOT NULL DEFAULT 0, - response_body MEDIUMBLOB, - expires_at DATETIME(3) NOT NULL, - created_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3), - updated_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3) ON UPDATE CURRENT_TIMESTAMP(3), - PRIMARY KEY (id), - UNIQUE INDEX uk_im_http_idempotency_scope (tenant_id, user_id, method, route, idempotency_key), - INDEX idx_im_http_idempotency_expiry (expires_at) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; +-- HTTP write idempotency is Redis-backed in the latest code. +DROP TABLE IF EXISTS meeting_im.im_http_idempotency; +DROP TABLE IF EXISTS tinode.im_http_idempotency; CREATE TABLE IF NOT EXISTS tinode.im_message_revisions ( id BIGINT NOT NULL AUTO_INCREMENT, @@ -1062,21 +1032,7 @@ SELECT id, 1, 999 FROM hotgo.im_tenant WHERE state = 1; -CREATE TABLE IF NOT EXISTS meeting_im.im_login_lockouts ( - id BIGINT NOT NULL, - user_id BIGINT, - account_hash CHAR(64) NOT NULL, - failed_attempts INT NOT NULL DEFAULT 0, - window_started_at DATETIME(3), - locked_until DATETIME(3), - last_attempt_at DATETIME(3), - created_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3), - updated_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3) ON UPDATE CURRENT_TIMESTAMP(3), - PRIMARY KEY (id), - UNIQUE INDEX uk_im_login_lockouts_account (account_hash), - INDEX idx_im_login_lockouts_user (user_id), - INDEX idx_im_login_lockouts_locked (locked_until) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; +DROP TABLE IF EXISTS meeting_im.im_login_lockouts; -- Current MySQL adapter schema version expected by server/db/mysql/adapter.go. UPDATE tinode.im_kvmeta SET `value` = 123 WHERE `key` = 'version'; diff --git a/docs/IM-群聊业务表设计.sql b/docs/IM-群聊业务表设计.sql index 4348ea5..e8fe044 100644 --- a/docs/IM-群聊业务表设计.sql +++ b/docs/IM-群聊业务表设计.sql @@ -244,20 +244,5 @@ CREATE TABLE IF NOT EXISTS im_group_admin_logs ( INDEX idx_im_group_admin_logs_object (object_type, object_id) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; --- 11. Group operation idempotency. --- This can later be replaced by a global HTTP idempotency table. -CREATE TABLE IF NOT EXISTS im_group_idempotency ( - id BIGINT NOT NULL AUTO_INCREMENT, - user_id BIGINT NOT NULL, - method VARCHAR(8) NOT NULL, - route VARCHAR(256) NOT NULL, - idempotency_key VARCHAR(128) NOT NULL, - body_hash CHAR(64) NOT NULL, - result_code INT NOT NULL DEFAULT 0, - result_body JSON, - created_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3), - updated_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3) ON UPDATE CURRENT_TIMESTAMP(3), - PRIMARY KEY (id), - UNIQUE INDEX uk_im_group_idempotency_request (user_id, method, route, idempotency_key), - INDEX idx_im_group_idempotency_created (created_at) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; +-- 11. Group operation idempotency is handled by Redis-backed HTTP idempotency. +DROP TABLE IF EXISTS im_group_idempotency; diff --git a/docs/sql/IM-Kafka基础表SQL.sql b/docs/sql/IM-Kafka基础表SQL.sql index 1f91efd..4f10291 100644 --- a/docs/sql/IM-Kafka基础表SQL.sql +++ b/docs/sql/IM-Kafka基础表SQL.sql @@ -2,32 +2,10 @@ -- Date: 2026-08-26 -- Scope: run once in every tenant business database, such as tinode. -- Notes: --- * Do not run in hotgo. Admin/HotGo side needs its own admin_command/outbox tables. --- * Tables do not contain tenant_id because each tenant has an independent database. +-- * Do not run in hotgo. Admin/HotGo side needs its own admin_command tables. +-- * Kafka consume idempotency/retry state is Redis-backed; MySQL only keeps manual dead letters. -CREATE TABLE IF NOT EXISTS im_kafka_consume_log ( - id BIGINT NOT NULL AUTO_INCREMENT, - event_id VARCHAR(96) NOT NULL, - consumer_group VARCHAR(128) NOT NULL, - source_topic VARCHAR(128) NOT NULL, - event_type VARCHAR(96) NOT NULL DEFAULT '', - biz_type VARCHAR(64) NOT NULL DEFAULT '', - biz_id VARCHAR(128) NOT NULL DEFAULT '', - status VARCHAR(16) NOT NULL DEFAULT 'processing', - retry_count INT NOT NULL DEFAULT 0, - last_reason VARCHAR(64) NOT NULL DEFAULT '', - last_error VARCHAR(1000) NOT NULL DEFAULT '', - processing_deadline DATETIME(3), - succeeded_at DATETIME(3), - dead_at DATETIME(3), - created_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3), - updated_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3) ON UPDATE CURRENT_TIMESTAMP(3), - PRIMARY KEY(id), - UNIQUE INDEX uk_im_kafka_consume_log_event_group(event_id, consumer_group), - INDEX idx_im_kafka_consume_log_status(status, processing_deadline), - INDEX idx_im_kafka_consume_log_biz(biz_type, biz_id), - INDEX idx_im_kafka_consume_log_topic(source_topic, id) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; +DROP TABLE IF EXISTS im_kafka_consume_log; CREATE TABLE IF NOT EXISTS im_kafka_dead_letters ( id BIGINT NOT NULL AUTO_INCREMENT, diff --git a/docs/sql/IM-共库核心分表-仅本地.sql b/docs/sql/IM-共库核心分表-仅本地.sql index bd10a7b..22ec632 100644 --- a/docs/sql/IM-共库核心分表-仅本地.sql +++ b/docs/sql/IM-共库核心分表-仅本地.sql @@ -44,19 +44,9 @@ ALTER TABLE `im_group_message_shards` ADD UNIQUE INDEX `uk_im_group_message_shards_table` (`tenant_id`, `table_name`), ADD INDEX `idx_im_group_message_shards_status` (`tenant_id`, `status`, `shard_month`); -ALTER TABLE `im_group_message_topic_shards` - DROP PRIMARY KEY, - DROP INDEX `idx_im_group_message_topic_shards_seq`, - DROP INDEX `idx_im_group_message_topic_shards_month`, - ADD PRIMARY KEY (`tenant_id`, `topic`, `shard_month`), - ADD INDEX `idx_im_group_message_topic_shards_seq` (`tenant_id`, `topic`, `max_seq`), - ADD INDEX `idx_im_group_message_topic_shards_month` (`tenant_id`, `shard_month`, `topic`); +DROP TABLE IF EXISTS `im_group_message_topic_shards`; -ALTER TABLE `im_group_message_dedup` - DROP PRIMARY KEY, - DROP INDEX `idx_im_group_message_dedup_topic_seq`, - ADD PRIMARY KEY (`tenant_id`, `topic`, `from_user`, `client_msg_id`), - ADD INDEX `idx_im_group_message_dedup_topic_seq` (`tenant_id`, `topic`, `seqid`); +DROP TABLE IF EXISTS `im_group_message_dedup`; -- 3. 创建好友与单聊消息分表并迁移现有数据。 CREATE TABLE IF NOT EXISTS `im_001_friendships` LIKE `im_friendships`; @@ -146,17 +136,7 @@ SELECT 2,`shard_month`,CONCAT('im_002_group_messages_', `shard_month`),`shard_st FROM `meeting_im`.`im_group_message_shards` ON DUPLICATE KEY UPDATE `table_name`=VALUES(`table_name`),`status`=VALUES(`status`); -INSERT IGNORE INTO `im_group_message_topic_shards` SELECT * FROM `meeting_im`.`im_group_message_topic_shards`; -INSERT IGNORE INTO `im_group_message_dedup` SELECT * FROM `meeting_im`.`im_group_message_dedup`; - --- 旧非月表迁移后补齐 Topic 路由，并按物理表实际数据校准分片统计。 -INSERT INTO `im_group_message_topic_shards` -(`tenant_id`,`topic`,`shard_month`,`min_seq`,`max_seq`,`message_count`,`min_created_at`,`max_created_at`) -SELECT 1,`topic`,'202608',MIN(`seqid`),MAX(`seqid`),COUNT(*),MIN(`created_at`),MAX(`created_at`) -FROM `im_001_group_messages_202608` -GROUP BY `topic` -ON DUPLICATE KEY UPDATE `min_seq`=VALUES(`min_seq`),`max_seq`=VALUES(`max_seq`), -`message_count`=VALUES(`message_count`),`min_created_at`=VALUES(`min_created_at`),`max_created_at`=VALUES(`max_created_at`); +-- Topic-to-month routes are Redis-backed and are rebuilt by runtime reads. UPDATE `im_group_message_shards` AS s SET s.`row_count`=(SELECT COUNT(*) FROM `im_001_group_messages_202608`), diff --git a/docs/sql/IM-共库核心分表-测试预演库.sql b/docs/sql/IM-共库核心分表-测试预演库.sql index 65ce736..82e005a 100644 --- a/docs/sql/IM-共库核心分表-测试预演库.sql +++ b/docs/sql/IM-共库核心分表-测试预演库.sql @@ -59,19 +59,9 @@ ALTER TABLE `im_group_message_shards` ADD UNIQUE INDEX `uk_im_group_message_shards_table` (`tenant_id`,`table_name`), ADD INDEX `idx_im_group_message_shards_status` (`tenant_id`,`status`,`shard_month`); -ALTER TABLE `im_group_message_topic_shards` - DROP PRIMARY KEY, - DROP INDEX `idx_im_group_message_topic_shards_seq`, - DROP INDEX `idx_im_group_message_topic_shards_month`, - ADD PRIMARY KEY (`tenant_id`,`topic`,`shard_month`), - ADD INDEX `idx_im_group_message_topic_shards_seq` (`tenant_id`,`topic`,`max_seq`), - ADD INDEX `idx_im_group_message_topic_shards_month` (`tenant_id`,`shard_month`,`topic`); +DROP TABLE IF EXISTS `im_group_message_topic_shards`; -ALTER TABLE `im_group_message_dedup` - DROP PRIMARY KEY, - DROP INDEX `idx_im_group_message_dedup_topic_seq`, - ADD PRIMARY KEY (`tenant_id`,`topic`,`from_user`,`client_msg_id`), - ADD INDEX `idx_im_group_message_dedup_topic_seq` (`tenant_id`,`topic`,`seqid`); +DROP TABLE IF EXISTS `im_group_message_dedup`; -- 3. 创建商户 2 的好友与单聊消息分表并复制快照数据。 CREATE TABLE `im_002_friendships` LIKE `im_friendships`; @@ -208,21 +198,7 @@ SELECT 1,`shard_month`,CONCAT('im_001_group_messages_',`shard_month`),`shard_sta `status`,`row_count`,`min_created_at`,`max_created_at`,`created_at`,`updated_at` FROM `tinode`.`im_group_message_shards`; -INSERT INTO `im_group_message_topic_shards` ( - `tenant_id`,`topic`,`shard_month`,`min_seq`,`max_seq`,`message_count`, - `min_created_at`,`max_created_at`,`created_at`,`updated_at` -) -SELECT 1,`topic`,`shard_month`,`min_seq`,`max_seq`,`message_count`, - `min_created_at`,`max_created_at`,`created_at`,`updated_at` -FROM `tinode`.`im_group_message_topic_shards`; - -INSERT INTO `im_group_message_dedup` ( - `tenant_id`,`topic`,`from_user`,`client_msg_id`,`seqid`,`shard_month`, - `message_id`,`status`,`expires_at`,`created_at`,`updated_at` -) -SELECT 1,`topic`,`from_user`,`client_msg_id`,`seqid`,`shard_month`, - `message_id`,`status`,`expires_at`,`created_at`,`updated_at` -FROM `tinode`.`im_group_message_dedup`; +-- Topic-to-month routes are Redis-backed and are rebuilt by runtime reads. -- 以实际复制后的物理表为准刷新统计，避免源库路由统计滞后。 DELIMITER $$ diff --git a/docs/sql/IM-共库运行态缺口修复SQL.sql b/docs/sql/IM-共库运行态缺口修复SQL.sql index 6962c2a..b554070 100644 --- a/docs/sql/IM-共库运行态缺口修复SQL.sql +++ b/docs/sql/IM-共库运行态缺口修复SQL.sql @@ -29,52 +29,19 @@ SELECT id, 1, 999 FROM hotgo.im_tenant WHERE state = 1; -CREATE TABLE IF NOT EXISTS im_login_lockouts ( - id BIGINT NOT NULL, - user_id BIGINT NULL, - account_hash CHAR(64) NOT NULL, - failed_attempts INT NOT NULL DEFAULT 0, - window_started_at DATETIME(3) NULL, - locked_until DATETIME(3) NULL, - last_attempt_at DATETIME(3) NULL, - created_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3), - updated_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3) - ON UPDATE CURRENT_TIMESTAMP(3), - PRIMARY KEY (id), - UNIQUE INDEX uk_im_login_lockouts_account (account_hash), - INDEX idx_im_login_lockouts_user (user_id), - INDEX idx_im_login_lockouts_locked (locked_until) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; +DROP TABLE IF EXISTS im_login_lockouts; --- HTTP write idempotency must be tenant scoped in the shared meeting_im DB. -CREATE TABLE IF NOT EXISTS im_http_idempotency ( - id BIGINT NOT NULL AUTO_INCREMENT, - tenant_id BIGINT NOT NULL DEFAULT 0, - user_id BIGINT NOT NULL, - method VARCHAR(8) NOT NULL, - route VARCHAR(512) NOT NULL, - idempotency_key VARCHAR(128) NOT NULL, - body_hash CHAR(64) NOT NULL, - state VARCHAR(16) NOT NULL DEFAULT 'processing', - response_status INT NOT NULL DEFAULT 0, - response_body MEDIUMBLOB, - expires_at DATETIME(3) NOT NULL, - created_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3), - updated_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3) - ON UPDATE CURRENT_TIMESTAMP(3), - PRIMARY KEY (id), - UNIQUE INDEX uk_im_http_idempotency_scope - (tenant_id, user_id, method, route, idempotency_key), - INDEX idx_im_http_idempotency_expiry (expires_at) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; - -ALTER TABLE im_http_idempotency - ADD COLUMN tenant_id BIGINT NOT NULL DEFAULT 0 AFTER id; -ALTER TABLE im_http_idempotency - DROP INDEX uk_im_http_idempotency_scope; -ALTER TABLE im_http_idempotency - ADD UNIQUE INDEX uk_im_http_idempotency_scope - (tenant_id, user_id, method, route, idempotency_key); +-- HTTP write idempotency is Redis-backed. Keep it out of MySQL. +DROP TABLE IF EXISTS im_http_idempotency; +DROP TABLE IF EXISTS im_group_idempotency; +DROP TABLE IF EXISTS im_group_message_dedup; +DROP TABLE IF EXISTS im_group_message_topic_shards; +DROP TABLE IF EXISTS im_kafka_outbox; +DROP TABLE IF EXISTS im_kafka_consume_log; +DROP TABLE IF EXISTS im_business_events_outbox; +DROP TABLE IF EXISTS im_group_events_outbox; +DROP TABLE IF EXISTS im_login_failure_logs; +DROP TABLE IF EXISTS im_action_tokens; -- User identity indexes must be unique inside one tenant, not globally. ALTER TABLE im_users diff --git a/docs/sql/IM-后台消息与登录审计SQL.sql b/docs/sql/IM-后台消息与登录审计SQL.sql index 1c4d413..2a5d8ca 100644 --- a/docs/sql/IM-后台消息与登录审计SQL.sql +++ b/docs/sql/IM-后台消息与登录审计SQL.sql @@ -30,50 +30,9 @@ CREATE TABLE IF NOT EXISTS im_user_login_logs ( INDEX idx_im_user_login_request (request_id) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; -CREATE TABLE IF NOT EXISTS im_login_failure_logs ( - id BIGINT NOT NULL AUTO_INCREMENT, - user_id BIGINT NULL, - account_type VARCHAR(24) NOT NULL DEFAULT '', - account_hash CHAR(64) NOT NULL DEFAULT '', - account_mask VARCHAR(128) NOT NULL DEFAULT '', - failure_type VARCHAR(32) NOT NULL, - reason VARCHAR(96) NOT NULL DEFAULT '', - detail VARCHAR(500) NOT NULL DEFAULT '', - ip VARCHAR(64) NOT NULL DEFAULT '', - region VARCHAR(128) NOT NULL DEFAULT '', - device_id VARCHAR(255) NOT NULL DEFAULT '', - platform VARCHAR(32) NOT NULL DEFAULT 'unknown', - app_version VARCHAR(32) NOT NULL DEFAULT '', - user_agent VARCHAR(512) NOT NULL DEFAULT '', - risk_level VARCHAR(16) NOT NULL DEFAULT 'medium', - risk_action VARCHAR(24) NOT NULL DEFAULT 'audit_only', - request_id VARCHAR(64) NOT NULL DEFAULT '', - attempt_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3), - metadata JSON NULL, - created_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3), - PRIMARY KEY (id), - INDEX idx_im_login_failure_account_time (account_hash, attempt_at DESC, id DESC), - INDEX idx_im_login_failure_user_time (user_id, attempt_at DESC, id DESC), - INDEX idx_im_login_failure_ip_time (ip, attempt_at DESC, id DESC), - INDEX idx_im_login_failure_type_time (failure_type, attempt_at DESC, id DESC), - INDEX idx_im_login_failure_request (request_id) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; +DROP TABLE IF EXISTS im_login_failure_logs; -CREATE TABLE IF NOT EXISTS im_login_lockouts ( - id BIGINT NOT NULL, - user_id BIGINT NULL, - account_hash CHAR(64) NOT NULL, - failed_attempts INT NOT NULL DEFAULT 0, - window_started_at DATETIME(3) NULL, - locked_until DATETIME(3) NULL, - last_attempt_at DATETIME(3) NULL, - created_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3), - updated_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3) ON UPDATE CURRENT_TIMESTAMP(3), - PRIMARY KEY (id), - UNIQUE INDEX uk_im_login_lockouts_account (account_hash), - INDEX idx_im_login_lockouts_user (user_id), - INDEX idx_im_login_lockouts_locked (locked_until) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; +DROP TABLE IF EXISTS im_login_lockouts; CREATE TABLE IF NOT EXISTS im_message_admin_actions ( id BIGINT NOT NULL AUTO_INCREMENT, diff --git a/docs/sql/IM-群消息按月分表SQL.sql b/docs/sql/IM-群消息按月分表SQL.sql index 4624f0f..d3bdffc 100644 --- a/docs/sql/IM-群消息按月分表SQL.sql +++ b/docs/sql/IM-群消息按月分表SQL.sql @@ -17,36 +17,9 @@ CREATE TABLE IF NOT EXISTS im_group_message_shards ( KEY idx_im_group_message_shards_status (status, shard_month) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; -CREATE TABLE IF NOT EXISTS im_group_message_topic_shards ( - topic VARCHAR(64) NOT NULL, - shard_month CHAR(6) NOT NULL, - min_seq BIGINT NOT NULL, - max_seq BIGINT NOT NULL, - message_count BIGINT NOT NULL DEFAULT 0, - min_created_at DATETIME(3) NOT NULL, - max_created_at DATETIME(3) NOT NULL, - created_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3), - updated_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3) ON UPDATE CURRENT_TIMESTAMP(3), - PRIMARY KEY (topic, shard_month), - KEY idx_im_group_message_topic_shards_seq (topic, max_seq), - KEY idx_im_group_message_topic_shards_month (shard_month, topic) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; +DROP TABLE IF EXISTS im_group_message_topic_shards; -CREATE TABLE IF NOT EXISTS im_group_message_dedup ( - topic VARCHAR(64) NOT NULL, - from_user BIGINT NOT NULL, - client_msg_id VARCHAR(64) NOT NULL, - seqid BIGINT NOT NULL, - shard_month CHAR(6) NOT NULL, - message_id BIGINT NULL, - status VARCHAR(16) NOT NULL DEFAULT 'pending', - expires_at DATETIME(3) NOT NULL, - created_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3), - updated_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3) ON UPDATE CURRENT_TIMESTAMP(3), - PRIMARY KEY (topic, from_user, client_msg_id), - KEY idx_im_group_message_dedup_expires (expires_at), - KEY idx_im_group_message_dedup_topic_seq (topic, seqid) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; +DROP TABLE IF EXISTS im_group_message_dedup; CREATE TABLE IF NOT EXISTS im_group_messages_202609 ( message_id BIGINT NOT NULL, @@ -104,4 +77,4 @@ ON DUPLICATE KEY UPDATE -- Historical migration is deliberately separate from schema deployment. -- Backfill im_group_messages into the matching month table in bounded batches, --- then populate im_group_message_topic_shards before removing legacy reads. +-- then warm Redis topic routes before removing legacy reads. diff --git a/server/auth/token/auth_token_test.go b/server/auth/token/auth_token_test.go index 31b51ff..2f7e6ab 100644 --- a/server/auth/token/auth_token_test.go +++ b/server/auth/token/auth_token_test.go @@ -79,8 +79,11 @@ func (s testSecurityStore) CountUnreadSystemNotificationsByEventPrefix(types.Ten return 0, s.err } -func (s testSecurityStore) MarkSystemNotificationRead(types.TenantID, types.Uid, string, time.Time) error { - return s.err +func (s testSecurityStore) MarkSystemNotificationRead(_ types.TenantID, _ types.Uid, eventID string, readAt time.Time) (*types.SystemNotification, error) { + if s.err != nil { + return nil, s.err + } + return &types.SystemNotification{EventID: eventID, EventType: "system.notice", Title: "通知", Summary: "通知", ReadAt: &readAt, CreatedAt: readAt}, nil } func newTestAuthenticator(t *testing.T) *authenticator { diff --git a/server/auth_audit_test.go b/server/auth_audit_test.go index 1aee4c6..9087c95 100644 --- a/server/auth_audit_test.go +++ b/server/auth_audit_test.go @@ -90,16 +90,15 @@ func TestGroupMessageQuerySupportsOperationalFilters(t *testing.T) { } func TestLoginAuditQuerySpecsExposeUserIDAsString(t *testing.T) { - for label, spec := range map[string]*internalQuerySpec{ - "user-login-logs": internalTenantResourceSpecs["user-login-logs"], - "login-failure-logs": internalTenantResourceSpecs["login-failure-logs"], - } { - if spec == nil { - t.Fatalf("%s query spec missing", label) - } - if !strings.Contains(spec.Select, "CAST(user_id AS CHAR) AS user_id") { - t.Fatalf("%s query must expose 64-bit user IDs as strings: %s", label, spec.Select) - } + spec := internalTenantResourceSpecs["user-login-logs"] + if spec == nil { + t.Fatal("user-login-logs query spec missing") + } + if !strings.Contains(spec.Select, "CAST(user_id AS CHAR) AS user_id") { + t.Fatalf("user-login-logs query must expose 64-bit user IDs as strings: %s", spec.Select) + } + if internalTenantResourceSpecs["login-failure-logs"] != nil { + t.Fatal("login-failure-logs query spec must be removed with im_login_failure_logs") } } diff --git a/server/db/adapter.go b/server/db/adapter.go index 05720b4..27f2dab 100644 --- a/server/db/adapter.go +++ b/server/db/adapter.go @@ -261,14 +261,8 @@ type TenantRoutedAdapter interface { } // TenantHTTPInfrastructureAdapter is the optional durable infrastructure used -// by product HTTP middleware and online business-event delivery. +// by Kafka dead-letter handling. type TenantHTTPInfrastructureAdapter interface { - TenantHTTPIdempotencyBegin(t.TenantID, *t.HTTPIdempotencyRecord) (*t.HTTPIdempotencyRecord, bool, error) - TenantHTTPIdempotencyComplete(t.TenantID, *t.HTTPIdempotencyRecord) error - TenantHTTPIdempotencyAbort(t.TenantID, *t.HTTPIdempotencyRecord) error - TenantKafkaConsumeBegin(t.TenantID, *t.KafkaConsumeRecord, time.Time) (*t.KafkaConsumeRecord, bool, error) - TenantKafkaConsumeSucceed(t.TenantID, string, string, time.Time) error - TenantKafkaConsumeFail(t.TenantID, string, string, string, string, time.Time) (*t.KafkaConsumeRecord, error) TenantKafkaConsumeDead(t.TenantID, *t.KafkaDeadLetter, time.Time) error TenantKafkaDeadLetterList(t.TenantID, t.KafkaDeadLetterListOptions) (*t.KafkaDeadLetterPage, error) TenantKafkaDeadLetterGet(t.TenantID, string) (*t.KafkaDeadLetter, error) @@ -476,7 +470,7 @@ type TenantBusinessAdapter interface { TenantSystemNotificationList(t.TenantID, t.Uid, string, int) ([]t.SystemNotification, string, error) TenantSystemNotificationUnreadCount(t.TenantID, t.Uid) (int, error) TenantSystemNotificationUnreadCountByEventPrefix(t.TenantID, t.Uid, string) (int, error) - TenantSystemNotificationMarkRead(t.TenantID, t.Uid, string, time.Time) error + TenantSystemNotificationMarkRead(t.TenantID, t.Uid, string, time.Time) (*t.SystemNotification, error) TenantFriendRequestCreate(t.TenantID, *t.FriendRequest) (*t.FriendRequest, error) TenantFriendRequestList(t.TenantID, t.Uid, string, string, string, int) ([]t.FriendRequestListItem, string, error) diff --git a/server/db/mysql/adapter.go b/server/db/mysql/adapter.go index 45add0b..58666e4 100644 --- a/server/db/mysql/adapter.go +++ b/server/db/mysql/adapter.go @@ -1148,25 +1148,6 @@ func createIMSecurityTables(tx *sql.Tx) error { )`); err != nil { return err } - if _, err := tx.Exec( - `CREATE TABLE im_action_tokens( - id BIGINT NOT NULL AUTO_INCREMENT, - user_id BIGINT NOT NULL, - token_id VARCHAR(64) NOT NULL, - action VARCHAR(64) NOT NULL, - target VARCHAR(128) NOT NULL DEFAULT '', - token_hash CHAR(64) NOT NULL, - state SMALLINT NOT NULL, - expires_at DATETIME(3) NOT NULL, - consumed_at DATETIME(3), - created_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3), - updated_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3) ON UPDATE CURRENT_TIMESTAMP(3), - PRIMARY KEY(id), - UNIQUE INDEX uk_im_action_tokens_token(token_id), - INDEX idx_im_action_tokens_user_action(user_id, action, target, state, expires_at) - )`); err != nil { - return err - } _, err := tx.Exec( `CREATE TABLE im_youth_mode_settings( id BIGINT NOT NULL, diff --git a/server/db/mysql/admin_governance_schema.go b/server/db/mysql/admin_governance_schema.go index 9ce9a92..f3d2bea 100644 --- a/server/db/mysql/admin_governance_schema.go +++ b/server/db/mysql/admin_governance_schema.go @@ -21,33 +21,6 @@ func ensureIMAdminGovernanceTables(exec sqlExecutor) error { INDEX idx_im_user_login_device_time(device_id(64), login_at DESC), INDEX idx_im_user_login_request(request_id) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci`, - `CREATE TABLE IF NOT EXISTS im_login_failure_logs ( - id BIGINT NOT NULL AUTO_INCREMENT, user_id BIGINT, - account_type VARCHAR(24) NOT NULL DEFAULT '', account_hash CHAR(64) NOT NULL DEFAULT '', - account_mask VARCHAR(128) NOT NULL DEFAULT '', failure_type VARCHAR(32) NOT NULL, - reason VARCHAR(96) NOT NULL DEFAULT '', detail VARCHAR(500) NOT NULL DEFAULT '', - ip VARCHAR(64) NOT NULL DEFAULT '', region VARCHAR(128) NOT NULL DEFAULT '', - device_id VARCHAR(255) NOT NULL DEFAULT '', platform VARCHAR(32) NOT NULL DEFAULT 'unknown', - app_version VARCHAR(32) NOT NULL DEFAULT '', user_agent VARCHAR(512) NOT NULL DEFAULT '', - risk_level VARCHAR(16) NOT NULL DEFAULT 'medium', risk_action VARCHAR(24) NOT NULL DEFAULT 'audit_only', - request_id VARCHAR(64) NOT NULL DEFAULT '', attempt_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3), - metadata JSON, created_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3), - PRIMARY KEY(id), INDEX idx_im_login_failure_account_time(account_hash, attempt_at DESC, id DESC), - INDEX idx_im_login_failure_user_time(user_id, attempt_at DESC, id DESC), - INDEX idx_im_login_failure_ip_time(ip, attempt_at DESC, id DESC), - INDEX idx_im_login_failure_type_time(failure_type, attempt_at DESC, id DESC), - INDEX idx_im_login_failure_request(request_id) - ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci`, - `CREATE TABLE IF NOT EXISTS im_login_lockouts ( - id BIGINT NOT NULL, user_id BIGINT, - account_hash CHAR(64) NOT NULL, - failed_attempts INT NOT NULL DEFAULT 0, - window_started_at DATETIME(3), locked_until DATETIME(3), last_attempt_at DATETIME(3), - created_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3), - updated_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3) ON UPDATE CURRENT_TIMESTAMP(3), - PRIMARY KEY(id), UNIQUE INDEX uk_im_login_lockouts_account(account_hash), - INDEX idx_im_login_lockouts_user(user_id), INDEX idx_im_login_lockouts_locked(locked_until) - ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci`, `CREATE TABLE IF NOT EXISTS im_message_admin_actions ( id BIGINT NOT NULL AUTO_INCREMENT, action_id VARCHAR(96) NOT NULL, command_id VARCHAR(96) NOT NULL, conversation_type VARCHAR(16) NOT NULL, diff --git a/server/db/mysql/admin_governance_schema_test.go b/server/db/mysql/admin_governance_schema_test.go index e96e5bf..76f9d18 100644 --- a/server/db/mysql/admin_governance_schema_test.go +++ b/server/db/mysql/admin_governance_schema_test.go @@ -13,15 +13,20 @@ func TestEnsureIMAdminGovernanceTablesUsesSelectedTenantDatabase(t *testing.T) { if err := ensureIMAdminGovernanceTables(exec); err != nil { t.Fatal(err) } - if len(exec.statements) != 5 { - t.Fatalf("created %d admin governance tables, want 5", len(exec.statements)) + if len(exec.statements) != 3 { + t.Fatalf("created %d admin governance tables, want 3", len(exec.statements)) } joined := strings.ToLower(strings.Join(exec.statements, "\n")) - for _, table := range []string{"im_user_login_logs", "im_login_failure_logs", "im_login_lockouts", "im_message_admin_actions", "im_user_ip_whitelists"} { + for _, table := range []string{"im_user_login_logs", "im_message_admin_actions", "im_user_ip_whitelists"} { if !strings.Contains(joined, table) { t.Fatalf("missing table %s", table) } } + for _, table := range []string{"im_login_failure_logs", "im_login_lockouts"} { + if strings.Contains(joined, table) { + t.Fatalf("hot login state table must not be created: %s", table) + } + } if strings.Contains(joined, "tinode.") || strings.Contains(joined, "create database") || strings.Contains(joined, "tenant_id") { t.Fatalf("admin governance schema must stay inside the selected tenant database: %s", joined) } diff --git a/server/db/mysql/group_message_shards.go b/server/db/mysql/group_message_shards.go index dd7dcdd..13f8ca0 100644 --- a/server/db/mysql/group_message_shards.go +++ b/server/db/mysql/group_message_shards.go @@ -25,24 +25,6 @@ func ensureGroupMessageShardInfrastructure(exec sqlExecutor, tenantID t.TenantID PRIMARY KEY(tenant_id, shard_month), UNIQUE INDEX uk_im_group_message_shards_table(tenant_id, table_name), INDEX idx_im_group_message_shards_status(tenant_id, status, shard_month) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci`, - `CREATE TABLE IF NOT EXISTS im_group_message_topic_shards ( - tenant_id BIGINT NOT NULL DEFAULT 0, topic VARCHAR(64) NOT NULL, shard_month CHAR(6) NOT NULL, - min_seq BIGINT NOT NULL, max_seq BIGINT NOT NULL, message_count BIGINT NOT NULL DEFAULT 0, - min_created_at DATETIME(3) NOT NULL, max_created_at DATETIME(3) NOT NULL, - created_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3), - updated_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3) ON UPDATE CURRENT_TIMESTAMP(3), - PRIMARY KEY(tenant_id, topic, shard_month), INDEX idx_im_group_message_topic_shards_seq(tenant_id, topic, max_seq), - INDEX idx_im_group_message_topic_shards_month(tenant_id, shard_month, topic) - ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci`, - `CREATE TABLE IF NOT EXISTS im_group_message_dedup ( - tenant_id BIGINT NOT NULL DEFAULT 0, topic VARCHAR(64) NOT NULL, from_user BIGINT NOT NULL, client_msg_id VARCHAR(64) NOT NULL, - seqid BIGINT NOT NULL, shard_month CHAR(6) NOT NULL, message_id BIGINT, - status VARCHAR(16) NOT NULL DEFAULT 'pending', expires_at DATETIME(3) NOT NULL, - created_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3), - updated_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3) ON UPDATE CURRENT_TIMESTAMP(3), - PRIMARY KEY(tenant_id, topic, from_user, client_msg_id), INDEX idx_im_group_message_dedup_expires(expires_at), - INDEX idx_im_group_message_dedup_topic_seq(tenant_id, topic, seqid) - ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci`, } for _, statement := range statements { if _, err := exec.Exec(statement); err != nil { @@ -129,22 +111,13 @@ func groupMessageShardTable(tenantID t.TenantID, month string) (string, bool) { func groupMessageTablesForTopic(q sqlx.QueryerContext, ctx context.Context, tenantID t.TenantID, topic string) ([]string, error) { routes, cached, err := store.GetGroupMessageShardRoutes(tenantID, topic) - redisAvailable := err == nil if err != nil { cached = false } if !cached { - routes = nil - if err = sqlx.SelectContext(ctx, q, &routes, - `SELECT shard_month AS month,min_seq,max_seq,message_count AS count - FROM im_group_message_topic_shards WHERE tenant_id=? AND topic=? ORDER BY shard_month DESC`, tenantID, topic); err != nil { + if routes, err = groupMessageAllShardRoutes(q, ctx, tenantID); err != nil { return nil, err } - if redisAvailable { - for _, route := range routes { - _ = store.PutGroupMessageShardRoute(tenantID, topic, route) - } - } } tables := make([]string, 0, len(routes)+1) for _, route := range routes { @@ -157,23 +130,13 @@ func groupMessageTablesForTopic(q sqlx.QueryerContext, ctx context.Context, tena func groupMessageTablesForSeq(q sqlx.QueryerContext, ctx context.Context, tenantID t.TenantID, topic string, seq int64) ([]string, error) { routes, cached, err := store.GetGroupMessageShardRoutes(tenantID, topic) - redisAvailable := err == nil if err != nil { cached = false } if !cached { - routes = nil - if err = sqlx.SelectContext(ctx, q, &routes, - `SELECT shard_month AS month,min_seq,max_seq,message_count AS count - FROM im_group_message_topic_shards WHERE tenant_id=? AND topic=? AND min_seq<=? AND max_seq>=? - ORDER BY shard_month DESC`, tenantID, topic, seq, seq); err != nil { + if routes, err = groupMessageAllShardRoutes(q, ctx, tenantID); err != nil { return nil, err } - if redisAvailable { - for _, route := range routes { - _ = store.PutGroupMessageShardRoute(tenantID, topic, route) - } - } } tables := make([]string, 0, 2) for _, route := range routes { @@ -186,6 +149,22 @@ func groupMessageTablesForSeq(q sqlx.QueryerContext, ctx context.Context, tenant return tables, nil } +func groupMessageAllShardRoutes(q sqlx.QueryerContext, ctx context.Context, tenantID t.TenantID) ([]store.GroupMessageShardRoute, error) { + var rows []struct { + Month string `db:"month"` + } + if err := sqlx.SelectContext(ctx, q, &rows, `SELECT shard_month AS month FROM im_group_message_shards WHERE tenant_id=? ORDER BY shard_month DESC`, tenantID); err != nil { + return nil, err + } + routes := make([]store.GroupMessageShardRoute, 0, len(rows)) + for _, row := range rows { + if validShardMonth(row.Month) { + routes = append(routes, store.GroupMessageShardRoute{Month: row.Month, MinSeq: 1, MaxSeq: 1<<62 - 1, Count: 0}) + } + } + return routes, nil +} + func groupMessageMonthFromTable(table string) string { const suffixLength = 6 if len(table) > suffixLength && validShardMonth(table[len(table)-suffixLength:]) { diff --git a/server/db/mysql/group_schema.go b/server/db/mysql/group_schema.go index 6ec412c..8537151 100644 --- a/server/db/mysql/group_schema.go +++ b/server/db/mysql/group_schema.go @@ -149,15 +149,6 @@ func ensureIMGroupBusinessTables(exec sqlExecutor) error { INDEX idx_im_group_admin_logs_topic_time(topic, id DESC), INDEX idx_im_group_admin_logs_operator(operator_id, id DESC), INDEX idx_im_group_admin_logs_object(object_type, object_id) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci`, - `CREATE TABLE IF NOT EXISTS im_group_idempotency ( - tenant_id BIGINT NOT NULL DEFAULT 0, id BIGINT NOT NULL AUTO_INCREMENT, user_id BIGINT NOT NULL, method VARCHAR(8) NOT NULL, - route VARCHAR(256) NOT NULL, idempotency_key VARCHAR(128) NOT NULL, body_hash CHAR(64) NOT NULL, - result_code INT NOT NULL DEFAULT 0, result_body JSON, - created_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3), - updated_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3) ON UPDATE CURRENT_TIMESTAMP(3), - PRIMARY KEY(id), UNIQUE INDEX uk_im_group_idempotency_request(user_id, method, route, idempotency_key), - INDEX idx_im_group_idempotency_created(created_at) - ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci`, `UPDATE im_topics SET public=JSON_SET(public, '$.fn', JSON_UNQUOTE(JSON_EXTRACT(public, '$.name'))) WHERE name LIKE 'grp%' AND JSON_VALID(public) AND COALESCE(JSON_UNQUOTE(JSON_EXTRACT(public, '$.fn')), '') = '' diff --git a/server/db/mysql/group_schema_test.go b/server/db/mysql/group_schema_test.go index 0eb962b..5620393 100644 --- a/server/db/mysql/group_schema_test.go +++ b/server/db/mysql/group_schema_test.go @@ -24,8 +24,8 @@ func TestEnsureIMGroupBusinessTablesUsesSelectedTenantDatabase(t *testing.T) { if err := ensureIMGroupBusinessTables(exec); err != nil { t.Fatal(err) } - if len(exec.statements) != 25 { - t.Fatalf("executed %d group schema statements, want 25", len(exec.statements)) + if len(exec.statements) != 22 { + t.Fatalf("executed %d group schema statements, want 22", len(exec.statements)) } for _, statement := range exec.statements { if strings.Contains(strings.ToLower(statement), "tinode.") || strings.Contains(strings.ToLower(statement), "create database") { @@ -36,6 +36,9 @@ func TestEnsureIMGroupBusinessTablesUsesSelectedTenantDatabase(t *testing.T) { if strings.Contains(joined, "im_group_events_outbox") { t.Fatal("group event outbox table must not be created") } + if strings.Contains(joined, "im_group_idempotency") { + t.Fatal("group idempotency table must not be created") + } for _, expected := range []string{ "primary key(id), unique index uk_im_group_settings_topic(topic)", "primary key(id), unique index uk_im_group_announcement_reads_announcement_user(announcement_id, user_id)", @@ -48,11 +51,17 @@ func TestEnsureIMGroupBusinessTablesUsesSelectedTenantDatabase(t *testing.T) { if !strings.Contains(joined, "json_set(public, '$.fn'") { t.Fatal("group schema is missing historical public.fn backfill") } - for _, expected := range []string{"im_group_message_shards", "im_group_message_topic_shards", "im_group_message_dedup", "primary key(message_id)"} { + for _, expected := range []string{"im_group_message_shards", "primary key(message_id)"} { if !strings.Contains(joined, expected) { t.Fatalf("group shard schema is missing %q", expected) } } + if strings.Contains(joined, "im_group_message_topic_shards") { + t.Fatal("group message topic shard routes must not be created in MySQL") + } + if strings.Contains(joined, "im_group_message_dedup") { + t.Fatal("group message dedup table must not be created") + } } func TestGroupMessageShardTableRejectsUnsafeMonth(t *testing.T) { @@ -71,11 +80,15 @@ func TestTenantPartitionTableNames(t *testing.T) { tenantID types.TenantID friends string messages string + topics string + members string + subs string + notices string group string }{ - {tenantID: 1, friends: "im_001_friendships", messages: "im_001_messages", group: "im_001_group_messages_202609"}, - {tenantID: 10, friends: "im_010_friendships", messages: "im_010_messages", group: "im_010_group_messages_202609"}, - {tenantID: 1000, friends: "im_1000_friendships", messages: "im_1000_messages", group: "im_1000_group_messages_202609"}, + {tenantID: 1, friends: "im_001_friendships", messages: "im_001_messages", topics: "im_001_topics", members: "im_001_group_members", subs: "im_001_subscriptions", notices: "im_001_system_notifications", group: "im_001_group_messages_202609"}, + {tenantID: 10, friends: "im_010_friendships", messages: "im_010_messages", topics: "im_010_topics", members: "im_010_group_members", subs: "im_010_subscriptions", notices: "im_010_system_notifications", group: "im_010_group_messages_202609"}, + {tenantID: 1000, friends: "im_1000_friendships", messages: "im_1000_messages", topics: "im_1000_topics", members: "im_1000_group_members", subs: "im_1000_subscriptions", notices: "im_1000_system_notifications", group: "im_1000_group_messages_202609"}, } for _, tc := range tests { if got := tenantFriendshipsTable(tc.tenantID); got != tc.friends { @@ -84,8 +97,34 @@ func TestTenantPartitionTableNames(t *testing.T) { if got := tenantMessagesTable(tc.tenantID); got != tc.messages { t.Fatalf("tenant %d message table: got %q, want %q", tc.tenantID, got, tc.messages) } + if got := tenantTopicsTable(tc.tenantID); got != tc.topics { + t.Fatalf("tenant %d topics table: got %q, want %q", tc.tenantID, got, tc.topics) + } + if got := tenantGroupMembersTable(tc.tenantID); got != tc.members { + t.Fatalf("tenant %d group members table: got %q, want %q", tc.tenantID, got, tc.members) + } + if got := tenantSubscriptionsTable(tc.tenantID); got != tc.subs { + t.Fatalf("tenant %d subscriptions table: got %q, want %q", tc.tenantID, got, tc.subs) + } + if got := tenantSystemNotificationsTable(tc.tenantID); got != tc.notices { + t.Fatalf("tenant %d system notifications table: got %q, want %q", tc.tenantID, got, tc.notices) + } if got, ok := tenantGroupMessagesTable(tc.tenantID, "202609"); !ok || got != tc.group { t.Fatalf("tenant %d group message table: got %q, ok=%v, want %q", tc.tenantID, got, ok, tc.group) } } } + +func TestTenantShardSQL(t *testing.T) { + query := tenantShardSQL(7, "SELECT * FROM im_topics t JOIN im_group_members gm ON gm.topic=t.name JOIN im_subscriptions s ON s.topic=gm.topic; INSERT INTO im_system_notifications(event_id) VALUES(?)") + for _, expected := range []string{"im_007_topics", "im_007_group_members", "im_007_subscriptions", "im_007_system_notifications"} { + if !strings.Contains(query, expected) { + t.Fatalf("tenant shard query is missing %q: %s", expected, query) + } + } + for _, legacy := range []string{" im_topics", " im_group_members", " im_subscriptions", " im_system_notifications"} { + if strings.Contains(query, legacy) { + t.Fatalf("tenant shard query still references %q: %s", legacy, query) + } + } +} diff --git a/server/db/mysql/infrastructure_schema.go b/server/db/mysql/infrastructure_schema.go index e4389dc..7c1a8e2 100644 --- a/server/db/mysql/infrastructure_schema.go +++ b/server/db/mysql/infrastructure_schema.go @@ -5,33 +5,6 @@ package mysql func ensureIMHTTPInfrastructureTables(exec sqlExecutor) error { statements := []string{ - `CREATE TABLE IF NOT EXISTS im_http_idempotency ( - id BIGINT NOT NULL AUTO_INCREMENT, tenant_id BIGINT NOT NULL DEFAULT 0, user_id BIGINT NOT NULL, - method VARCHAR(8) NOT NULL, route VARCHAR(512) NOT NULL, - idempotency_key VARCHAR(128) NOT NULL, body_hash CHAR(64) NOT NULL, - state VARCHAR(16) NOT NULL DEFAULT 'processing', response_status INT NOT NULL DEFAULT 0, - response_body MEDIUMBLOB, expires_at DATETIME(3) NOT NULL, - created_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3), - updated_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3) ON UPDATE CURRENT_TIMESTAMP(3), - PRIMARY KEY(id), - UNIQUE INDEX uk_im_http_idempotency_scope(tenant_id, user_id, method, route, idempotency_key), - INDEX idx_im_http_idempotency_expiry(expires_at) - ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci`, - `CREATE TABLE IF NOT EXISTS im_kafka_consume_log ( - id BIGINT NOT NULL AUTO_INCREMENT, event_id VARCHAR(96) NOT NULL, - consumer_group VARCHAR(128) NOT NULL, source_topic VARCHAR(128) NOT NULL, - event_type VARCHAR(96) NOT NULL DEFAULT '', biz_type VARCHAR(64) NOT NULL DEFAULT '', - biz_id VARCHAR(128) NOT NULL DEFAULT '', status VARCHAR(16) NOT NULL DEFAULT 'processing', - retry_count INT NOT NULL DEFAULT 0, last_reason VARCHAR(64) NOT NULL DEFAULT '', - last_error VARCHAR(1000) NOT NULL DEFAULT '', processing_deadline DATETIME(3), - succeeded_at DATETIME(3), dead_at DATETIME(3), - created_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3), - updated_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3) ON UPDATE CURRENT_TIMESTAMP(3), - PRIMARY KEY(id), UNIQUE INDEX uk_im_kafka_consume_log_event_group(event_id, consumer_group), - INDEX idx_im_kafka_consume_log_status(status, processing_deadline), - INDEX idx_im_kafka_consume_log_biz(biz_type, biz_id), - INDEX idx_im_kafka_consume_log_topic(source_topic, id) - ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci`, `CREATE TABLE IF NOT EXISTS im_kafka_dead_letters ( id BIGINT NOT NULL AUTO_INCREMENT, dead_id VARCHAR(96) NOT NULL, event_id VARCHAR(96) NOT NULL, parent_event_id VARCHAR(96) NOT NULL DEFAULT '', @@ -82,15 +55,5 @@ func ensureIMHTTPInfrastructureTables(exec sqlExecutor) error { return err } } - migrations := []ignoreSQLError{ - {`ALTER TABLE im_http_idempotency ADD COLUMN tenant_id BIGINT NOT NULL DEFAULT 0 AFTER id`, 1060}, - {`ALTER TABLE im_http_idempotency DROP INDEX uk_im_http_idempotency_scope`, 1091}, - {`ALTER TABLE im_http_idempotency ADD UNIQUE INDEX uk_im_http_idempotency_scope(tenant_id, user_id, method, route, idempotency_key)`, 1061}, - } - for _, migration := range migrations { - if err := execIgnoreMySQLError(exec, migration.sql, migration.code); err != nil { - return err - } - } return nil } diff --git a/server/db/mysql/infrastructure_schema_test.go b/server/db/mysql/infrastructure_schema_test.go index e92786d..bfcbf60 100644 --- a/server/db/mysql/infrastructure_schema_test.go +++ b/server/db/mysql/infrastructure_schema_test.go @@ -12,13 +12,11 @@ func TestEnsureIMHTTPInfrastructureTablesUsesSelectedTenantDatabase(t *testing.T if err := ensureIMHTTPInfrastructureTables(exec); err != nil { t.Fatal(err) } - if len(exec.statements) != 9 { - t.Fatalf("executed %d infrastructure schema statements, want 9", len(exec.statements)) + if len(exec.statements) != 4 { + t.Fatalf("executed %d infrastructure schema statements, want 4", len(exec.statements)) } joined := strings.ToLower(strings.Join(exec.statements, "\n")) for _, table := range []string{ - "im_http_idempotency", - "im_kafka_consume_log", "im_kafka_dead_letters", "im_realname_appeals", "im_conversation_settings", @@ -31,12 +29,15 @@ func TestEnsureIMHTTPInfrastructureTablesUsesSelectedTenantDatabase(t *testing.T if strings.Contains(joined, "tinode.") || strings.Contains(joined, "create database") { t.Fatalf("infrastructure schema must stay inside the selected tenant database: %s", joined) } - if !strings.Contains(joined, "tenant_id") || !strings.Contains(joined, "uk_im_http_idempotency_scope(tenant_id, user_id") { - t.Fatalf("HTTP idempotency schema must be tenant scoped in shared DB mode: %s", joined) + if strings.Contains(joined, "im_http_idempotency") { + t.Fatalf("HTTP idempotency must not create MySQL tables: %s", joined) } if strings.Contains(joined, "im_kafka_outbox") { t.Fatalf("kafka outbox table must not be created: %s", joined) } + if strings.Contains(joined, "im_kafka_consume_log") { + t.Fatalf("kafka consume state must not create MySQL tables: %s", joined) + } if strings.Contains(joined, "im_business_events_outbox") { t.Fatalf("business event outbox table must not be created: %s", joined) } diff --git a/server/db/mysql/login_lockouts.go b/server/db/mysql/login_lockouts.go index 8984472..a776e8e 100644 --- a/server/db/mysql/login_lockouts.go +++ b/server/db/mysql/login_lockouts.go @@ -4,203 +4,45 @@ package mysql import ( - "database/sql" - "errors" "time" t "github.com/tinode/chat/server/store/types" ) -func (a *adapter) routedLoginLockoutAdapter(id t.TenantID) (*adapter, error) { - if err := validTenant(id); err != nil { - return nil, err - } - if a.tenantScopedDB { - return a, nil - } - tenantAdp, err := a.routedTenantAdapter(id) - if err != nil { - return nil, err - } - return tenantAdp, nil -} - -type loginLockoutRow struct { - ID int64 `db:"id"` - UserID sql.NullInt64 `db:"user_id"` - AccountHash string `db:"account_hash"` - FailedAttempts int `db:"failed_attempts"` - WindowStartAt sql.NullTime `db:"window_started_at"` - LockedUntil sql.NullTime `db:"locked_until"` - LastAttemptAt sql.NullTime `db:"last_attempt_at"` - UpdatedAt sql.NullTime `db:"updated_at"` -} - -func loginLockoutFromRow(id t.TenantID, row loginLockoutRow) *t.LoginLockout { - out := &t.LoginLockout{ - TenantID: id, - ID: row.ID, - AccountHash: row.AccountHash, - FailedAttempts: row.FailedAttempts, - } - if row.UserID.Valid { - out.UserID = row.UserID.Int64 - } - if row.WindowStartAt.Valid { - v := row.WindowStartAt.Time - out.WindowStartAt = &v - } - if row.LockedUntil.Valid { - v := row.LockedUntil.Time - out.LockedUntil = &v - } - if row.LastAttemptAt.Valid { - v := row.LastAttemptAt.Time - out.LastAttemptAt = &v - } - if row.UpdatedAt.Valid { - v := row.UpdatedAt.Time - out.UpdatedAt = &v - } - return out -} - -const loginLockoutColumns = `id,user_id,account_hash,failed_attempts,window_started_at,locked_until,last_attempt_at,updated_at` - -// TenantLoginLockoutGet reads the lockout counter for the given account hash. +// Login lockout counters are runtime security state and are stored in Redis by +// the store layer. The MySQL adapter keeps these methods only to satisfy older +// adapter interfaces while preventing accidental writes to im_login_lockouts. func (a *adapter) TenantLoginLockoutGet(id t.TenantID, accountHash string) (*t.LoginLockout, error) { - tenantAdp, err := a.routedLoginLockoutAdapter(id) - if err != nil { - return nil, err - } - if tenantAdp != a { - return tenantAdp.TenantLoginLockoutGet(id, accountHash) - } - if accountHash == "" { + if err := validTenant(id); err != nil || accountHash == "" { return nil, t.ErrMalformed } - ctx, cancel := a.getContext() - if cancel != nil { - defer cancel() - } - var row loginLockoutRow - err = a.db.GetContext(ctx, &row, - `SELECT `+loginLockoutColumns+` FROM im_login_lockouts WHERE account_hash=?`, accountHash) - if errors.Is(err, sql.ErrNoRows) { - return nil, t.ErrNotFound - } - if err != nil { - return nil, err - } - return loginLockoutFromRow(id, row), nil + return nil, t.ErrUnsupported } -// TenantLoginLockoutRecordFailure atomically inserts or increments the failure -// counter. windowCutoff is "now minus window seconds": an older window start -// resets the counter to 1. The updated row is returned to the caller. func (a *adapter) TenantLoginLockoutRecordFailure(id t.TenantID, accountHash string, userID *int64, windowCutoff, now time.Time) (*t.LoginLockout, error) { - tenantAdp, err := a.routedLoginLockoutAdapter(id) - if err != nil { - return nil, err - } - if tenantAdp != a { - return tenantAdp.TenantLoginLockoutRecordFailure(id, accountHash, userID, windowCutoff, now) - } - if accountHash == "" || now.IsZero() { + if err := validTenant(id); err != nil || accountHash == "" || now.IsZero() { return nil, t.ErrMalformed } - ctx, cancel := a.getContext() - if cancel != nil { - defer cancel() - } - rowID, err := nextSnowflakeID() - if err != nil { - return nil, err - } - // MySQL evaluates ON DUPLICATE KEY UPDATE assignments left to right, so - // failed_attempts is recomputed before window_started_at is overwritten. - _, err = a.db.ExecContext(ctx, `INSERT INTO im_login_lockouts - (id,user_id,account_hash,failed_attempts,window_started_at,last_attempt_at) - VALUES(?,?,?,1,?,?) - ON DUPLICATE KEY UPDATE - failed_attempts = IF(window_started_at IS NULL OR window_started_at < ?, 1, failed_attempts + 1), - window_started_at = IF(window_started_at IS NULL OR window_started_at < ?, ?, window_started_at), - last_attempt_at = ?, - user_id = COALESCE(user_id, ?)`, - rowID, userID, accountHash, now, now, - windowCutoff, windowCutoff, now, now, userID) - if err != nil { - return nil, err - } - var row loginLockoutRow - err = a.db.GetContext(ctx, &row, - `SELECT `+loginLockoutColumns+` FROM im_login_lockouts WHERE account_hash=?`, accountHash) - if err != nil { - return nil, err - } - return loginLockoutFromRow(id, row), nil + return nil, t.ErrUnsupported } -// TenantLoginLockoutLock sets locked_until unless a newer lock already covers now. func (a *adapter) TenantLoginLockoutLock(id t.TenantID, accountHash string, lockedUntil, now time.Time) error { - tenantAdp, err := a.routedLoginLockoutAdapter(id) - if err != nil { - return err - } - if tenantAdp != a { - return tenantAdp.TenantLoginLockoutLock(id, accountHash, lockedUntil, now) - } - if accountHash == "" || lockedUntil.IsZero() || now.IsZero() { + if err := validTenant(id); err != nil || accountHash == "" || lockedUntil.IsZero() || now.IsZero() { return t.ErrMalformed } - ctx, cancel := a.getContext() - if cancel != nil { - defer cancel() - } - _, err = a.db.ExecContext(ctx, - `UPDATE im_login_lockouts SET locked_until=?, updated_at=? WHERE account_hash=? AND (locked_until IS NULL OR locked_until < ?)`, - lockedUntil, now, accountHash, now) - return err + return t.ErrUnsupported } -// TenantLoginLockoutReset removes the counter after a successful login. func (a *adapter) TenantLoginLockoutReset(id t.TenantID, accountHash string) error { - tenantAdp, err := a.routedLoginLockoutAdapter(id) - if err != nil { - return err - } - if tenantAdp != a { - return tenantAdp.TenantLoginLockoutReset(id, accountHash) - } - if accountHash == "" { + if err := validTenant(id); err != nil || accountHash == "" { return t.ErrMalformed } - ctx, cancel := a.getContext() - if cancel != nil { - defer cancel() - } - _, err = a.db.ExecContext(ctx, `DELETE FROM im_login_lockouts WHERE account_hash=?`, accountHash) - return err + return t.ErrUnsupported } -// TenantLoginLockoutRelease clears the lock and zeroes the counter (admin unlock). func (a *adapter) TenantLoginLockoutRelease(id t.TenantID, accountHash string, now time.Time) error { - tenantAdp, err := a.routedLoginLockoutAdapter(id) - if err != nil { - return err - } - if tenantAdp != a { - return tenantAdp.TenantLoginLockoutRelease(id, accountHash, now) - } - if accountHash == "" || now.IsZero() { + if err := validTenant(id); err != nil || accountHash == "" || now.IsZero() { return t.ErrMalformed } - ctx, cancel := a.getContext() - if cancel != nil { - defer cancel() - } - _, err = a.db.ExecContext(ctx, - `UPDATE im_login_lockouts SET failed_attempts=0, window_started_at=NULL, locked_until=NULL, updated_at=? WHERE account_hash=?`, - now, accountHash) - return err + return t.ErrUnsupported } diff --git a/server/db/mysql/schema.sql b/server/db/mysql/schema.sql index 7deefd0..c51bb5e 100644 --- a/server/db/mysql/schema.sql +++ b/server/db/mysql/schema.sql @@ -12,7 +12,7 @@ # # - Shared tenant business database: meeting_im # - im_users, im_topics, im_subscriptions, im_messages and extension tables -# - im_http_idempotency for HTTP write idempotency +# - Redis short-state keys for HTTP write idempotency # - im_file_review_tasks for asynchronous content safety and malware checks # - business tables store tenant_id; high-volume tables are physically partitioned by tenant # diff --git a/server/db/mysql/tenant_adapter.go b/server/db/mysql/tenant_adapter.go index fb15bba..1dfb2bc 100644 --- a/server/db/mysql/tenant_adapter.go +++ b/server/db/mysql/tenant_adapter.go @@ -322,11 +322,11 @@ func messageHeadBool(head t.KVMap, key string) bool { func (a *adapter) tenantTopicBumpMutationVersionTx(ctx context.Context, tx *sqlx.Tx, id t.TenantID, topic string, at time.Time) error { if a.tenantScopedDB { - _, err := tx.ExecContext(ctx, "UPDATE im_topics SET mutation_version=mutation_version+1,updatedat=?,touchedat=? WHERE name=?", + _, err := tx.ExecContext(ctx, tenantShardSQL(id, "UPDATE im_topics SET mutation_version=mutation_version+1,updatedat=?,touchedat=? WHERE name=?"), at, at, topic) return err } - _, err := tx.ExecContext(ctx, "UPDATE im_topics SET mutation_version=mutation_version+1,updatedat=?,touchedat=? WHERE tenant_id=? AND name=?", + _, err := tx.ExecContext(ctx, tenantShardSQL(id, "UPDATE im_topics SET mutation_version=mutation_version+1,updatedat=?,touchedat=? WHERE tenant_id=? AND name=?"), at, at, id, topic) return err } @@ -634,7 +634,7 @@ func (a *adapter) TenantUserDelete(id t.TenantID, uid t.Uid, hard bool) error { }() var ownTopics []string - q := "SELECT name FROM im_topics WHERE tenant_id=? AND owner=?" + q := tenantShardSQL(id, "SELECT name FROM im_topics WHERE tenant_id=? AND owner=?") args := []any{id, decoded} if !hard { q += " AND state!=?" @@ -663,20 +663,20 @@ func (a *adapter) TenantUserDelete(id t.TenantID, uid t.Uid, hard bool) error { return err } if len(ownTopics) > 0 { - if _, err = tx.Exec("DELETE dl FROM im_dellog AS dl JOIN im_topics AS tp ON tp.tenant_id=dl.tenant_id AND tp.name=dl.topic WHERE tp.tenant_id=? AND tp.owner=?", id, decoded); err != nil { + if _, err = tx.Exec(tenantShardSQL(id, "DELETE dl FROM im_dellog AS dl JOIN im_topics AS tp ON tp.tenant_id=dl.tenant_id AND tp.name=dl.topic WHERE tp.tenant_id=? AND tp.owner=?"), id, decoded); err != nil { return err } - if _, err = tx.Exec(fmt.Sprintf("DELETE m FROM %s AS m JOIN im_topics AS tp ON tp.tenant_id=m.tenant_id AND tp.name=m.topic WHERE tp.tenant_id=? AND tp.owner=?", tenantMessagesTable(id)), id, decoded); err != nil { + if _, err = tx.Exec(tenantShardSQL(id, fmt.Sprintf("DELETE m FROM %s AS m JOIN im_topics AS tp ON tp.tenant_id=m.tenant_id AND tp.name=m.topic WHERE tp.tenant_id=? AND tp.owner=?", tenantMessagesTable(id))), id, decoded); err != nil { return err } - sqlq, params, _ := sqlx.In("DELETE FROM im_subscriptions WHERE tenant_id=? AND topic IN (?)", id, ownAll) + sqlq, params, _ := sqlx.In(tenantShardSQL(id, "DELETE FROM im_subscriptions WHERE tenant_id=? AND topic IN (?)"), id, ownAll) if _, err = tx.Exec(tx.Rebind(sqlq), params...); err != nil { return err } - if _, err = tx.Exec("DELETE tt FROM im_topictags AS tt JOIN im_topics AS tp ON tp.tenant_id=tt.tenant_id AND tp.name=tt.topic WHERE tp.tenant_id=? AND tp.owner=?", id, decoded); err != nil { + if _, err = tx.Exec(tenantShardSQL(id, "DELETE tt FROM im_topictags AS tt JOIN im_topics AS tp ON tp.tenant_id=tt.tenant_id AND tp.name=tt.topic WHERE tp.tenant_id=? AND tp.owner=?"), id, decoded); err != nil { return err } - if _, err = tx.Exec("DELETE FROM im_topics WHERE tenant_id=? AND owner=?", id, decoded); err != nil { + if _, err = tx.Exec(tenantShardSQL(id, "DELETE FROM im_topics WHERE tenant_id=? AND owner=?"), id, decoded); err != nil { return err } } @@ -694,23 +694,23 @@ func (a *adapter) TenantUserDelete(id t.TenantID, uid t.Uid, hard bool) error { return err } if len(ownAll) > 0 { - sqlq, params, _ := sqlx.In("UPDATE im_subscriptions SET updatedat=?,deletedat=? WHERE tenant_id=? AND topic IN (?)", now, now, id, ownAll) + sqlq, params, _ := sqlx.In(tenantShardSQL(id, "UPDATE im_subscriptions SET updatedat=?,deletedat=? WHERE tenant_id=? AND topic IN (?)"), now, now, id, ownAll) if _, err = tx.Exec(tx.Rebind(sqlq), params...); err != nil { return err } } - if _, err = tx.Exec("UPDATE im_topics SET updatedat=?,touchedat=?,state=?,stateat=? WHERE tenant_id=? AND owner=?", + if _, err = tx.Exec(tenantShardSQL(id, "UPDATE im_topics SET updatedat=?,touchedat=?,state=?,stateat=? WHERE tenant_id=? AND owner=?"), now, now, t.StateDeleted, now, id, decoded); err != nil { return err } - if _, err = tx.Exec("UPDATE im_topics AS tp JOIN im_subscriptions AS s ON tp.tenant_id=s.tenant_id AND tp.name=s.topic "+ + if _, err = tx.Exec(tenantShardSQL(id, "UPDATE im_topics AS tp JOIN im_subscriptions AS s ON tp.tenant_id=s.tenant_id AND tp.name=s.topic "+ "SET tp.updatedat=?,tp.touchedat=?,tp.state=?,tp.stateat=? WHERE tp.tenant_id=? AND tp.owner=0 AND s.userid=? AND tp.name LIKE 'p2p%'", - now, now, t.StateDeleted, now, id, decoded); err != nil { + ), now, now, t.StateDeleted, now, id, decoded); err != nil { return err } - if _, err = tx.Exec("UPDATE im_subscriptions AS s_one JOIN im_subscriptions AS s_two ON s_one.tenant_id=s_two.tenant_id AND s_one.topic=s_two.topic "+ + if _, err = tx.Exec(tenantShardSQL(id, "UPDATE im_subscriptions AS s_one JOIN im_subscriptions AS s_two ON s_one.tenant_id=s_two.tenant_id AND s_one.topic=s_two.topic "+ "SET s_two.updatedat=?,s_two.deletedat=? WHERE s_one.tenant_id=? AND s_one.userid=? AND s_one.topic LIKE 'p2p%'", - now, now, id, decoded); err != nil { + ), now, now, id, decoded); err != nil { return err } if _, err = tx.Exec("UPDATE im_users SET updatedat=?,state=?,stateat=? WHERE tenant_id=? AND id=?", @@ -729,13 +729,13 @@ func (a *adapter) tenantTopicStateForUser(tx *sqlx.Tx, tenantID t.TenantID, deco if now.IsZero() { now = t.TimeNow() } - if _, err := tx.Exec("UPDATE im_topics SET state=?,stateat=? WHERE tenant_id=? AND owner=? AND state!=?", + if _, err := tx.Exec(tenantShardSQL(tenantID, "UPDATE im_topics SET state=?,stateat=? WHERE tenant_id=? AND owner=? AND state!=?"), objState, now, tenantID, decodedUID, t.StateDeleted); err != nil { return err } - _, err := tx.Exec("UPDATE im_topics AS tp JOIN im_subscriptions AS s ON tp.tenant_id=s.tenant_id AND tp.name=s.topic "+ + _, err := tx.Exec(tenantShardSQL(tenantID, "UPDATE im_topics AS tp JOIN im_subscriptions AS s ON tp.tenant_id=s.tenant_id AND tp.name=s.topic "+ "SET tp.state=?,tp.stateat=? WHERE tp.tenant_id=? AND tp.owner=0 AND s.userid=? AND tp.state!=?", - objState, now, tenantID, decodedUID, t.StateDeleted) + ), objState, now, tenantID, decodedUID, t.StateDeleted) return err } @@ -822,10 +822,10 @@ func (a *adapter) TenantUserUnreadCount(id t.TenantID, uids ...t.Uid) (map[t.Uid if len(decoded) == 0 { return counts, nil } - q, args, _ := sqlx.In("SELECT s.userid,SUM(tp.seqid)-SUM(s.readseqid) AS unreadcount "+ + q, args, _ := sqlx.In(tenantShardSQL(id, "SELECT s.userid,SUM(tp.seqid)-SUM(s.readseqid) AS unreadcount "+ "FROM im_topics AS tp JOIN im_subscriptions AS s ON tp.tenant_id=s.tenant_id AND tp.name=s.topic "+ "WHERE s.tenant_id=? AND s.userid IN (?) AND s.deletedat IS NULL AND tp.state!=? "+ - "AND INSTR(s.modewant,'R')>0 AND INSTR(s.modegiven,'R')>0 GROUP BY s.userid", id, decoded, t.StateDeleted) + "AND INSTR(s.modewant,'R')>0 AND INSTR(s.modegiven,'R')>0 GROUP BY s.userid"), id, decoded, t.StateDeleted) ctx, cancel := a.getContext() if cancel != nil { defer cancel() @@ -886,9 +886,9 @@ func tenantTopicCreate(tx *sqlx.Tx, topic *t.Topic) error { if topic == nil || topic.TenantID.IsZero() { return t.ErrMalformed } - _, err := tx.Exec("INSERT INTO im_topics(tenant_id,createdat,updatedat,touchedat,state,name,usebt,owner,access,public,trusted,tags,aux) "+ + _, err := tx.Exec(tenantShardSQL(topic.TenantID, "INSERT INTO im_topics(tenant_id,createdat,updatedat,touchedat,state,name,usebt,owner,access,public,trusted,tags,aux) "+ "VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?)", - topic.TenantID, topic.CreatedAt, topic.UpdatedAt, topic.TouchedAt, topic.State, topic.Id, topic.UseBt, + ), topic.TenantID, topic.CreatedAt, topic.UpdatedAt, topic.TouchedAt, topic.State, topic.Id, topic.UseBt, decodeUID(t.ParseUid(topic.Owner)), topic.Access, common.ToJSON(topic.Public), common.ToJSON(topic.Trusted), topic.Tags, common.ToJSON(topic.Aux)) if err != nil { @@ -913,25 +913,25 @@ func createTenantSubscription(tx *sqlx.Tx, tenantID t.TenantID, sub *t.Subscript DeletedAt sql.NullTime `db:"deletedat"` }{} err := tx.Get(&existing, - "SELECT id,deletedat FROM im_subscriptions WHERE tenant_id=? AND topic=? AND userid=? ORDER BY deletedat IS NULL DESC,id ASC LIMIT 1", + tenantShardSQL(tenantID, "SELECT id,deletedat FROM im_subscriptions WHERE tenant_id=? AND topic=? AND userid=? ORDER BY deletedat IS NULL DESC,id ASC LIMIT 1"), tenantID, sub.Topic, decodedUID) if err != nil && err != sql.ErrNoRows { return err } if err == nil { if existing.DeletedAt.Valid { - _, err = tx.Exec("UPDATE im_subscriptions SET createdat=?,updatedat=?,deletedat=NULL,modeWant=?,modeGiven=?,"+ + _, err = tx.Exec(tenantShardSQL(tenantID, "UPDATE im_subscriptions SET createdat=?,updatedat=?,deletedat=NULL,modeWant=?,modeGiven=?,"+ "delid=0,recvseqid=0,readseqid=0,private=? WHERE tenant_id=? AND id=?", - sub.CreatedAt, sub.UpdatedAt, sub.ModeWant.String(), sub.ModeGiven.String(), jpriv, tenantID, existing.ID) + ), sub.CreatedAt, sub.UpdatedAt, sub.ModeWant.String(), sub.ModeGiven.String(), jpriv, tenantID, existing.ID) } else { - _, err = tx.Exec("UPDATE im_subscriptions SET updatedat=?,modeWant=?,modeGiven=?,private=? WHERE tenant_id=? AND id=?", + _, err = tx.Exec(tenantShardSQL(tenantID, "UPDATE im_subscriptions SET updatedat=?,modeWant=?,modeGiven=?,private=? WHERE tenant_id=? AND id=?"), sub.UpdatedAt, sub.ModeWant.String(), sub.ModeGiven.String(), jpriv, tenantID, existing.ID) } if err != nil { return err } if isOwner { - _, err = tx.Exec("UPDATE im_topics SET owner=? WHERE tenant_id=? AND name=?", decodedUID, tenantID, sub.Topic) + _, err = tx.Exec(tenantShardSQL(tenantID, "UPDATE im_topics SET owner=? WHERE tenant_id=? AND name=?"), decodedUID, tenantID, sub.Topic) } return err } @@ -941,30 +941,30 @@ func createTenantSubscription(tx *sqlx.Tx, tenantID t.TenantID, sub *t.Subscript return err } _, err = tx.Exec( - "INSERT INTO im_subscriptions(id,tenant_id,createdat,updatedat,deletedat,userid,topic,modeWant,modeGiven,private) "+ - "VALUES(?,?,?,?,NULL,?,?,?,?,?)", + tenantShardSQL(tenantID, "INSERT INTO im_subscriptions(id,tenant_id,createdat,updatedat,deletedat,userid,topic,modeWant,modeGiven,private) "+ + "VALUES(?,?,?,?,NULL,?,?,?,?,?)"), subID, tenantID, sub.CreatedAt, sub.UpdatedAt, decodedUID, sub.Topic, sub.ModeWant.String(), sub.ModeGiven.String(), jpriv) if isIDOutOfRange(err) { _, err = tx.Exec( - "INSERT INTO im_subscriptions(tenant_id,createdat,updatedat,deletedat,userid,topic,modeWant,modeGiven,private) "+ - "VALUES(?,?,?,NULL,?,?,?,?,?)", + tenantShardSQL(tenantID, "INSERT INTO im_subscriptions(tenant_id,createdat,updatedat,deletedat,userid,topic,modeWant,modeGiven,private) "+ + "VALUES(?,?,?,NULL,?,?,?,?,?)"), tenantID, sub.CreatedAt, sub.UpdatedAt, decodedUID, sub.Topic, sub.ModeWant.String(), sub.ModeGiven.String(), jpriv) } if err != nil && isDupe(err) { if undelete { - _, err = tx.Exec("UPDATE im_subscriptions SET createdat=?,updatedat=?,deletedat=NULL,modeWant=?,modeGiven=?,"+ + _, err = tx.Exec(tenantShardSQL(tenantID, "UPDATE im_subscriptions SET createdat=?,updatedat=?,deletedat=NULL,modeWant=?,modeGiven=?,"+ "delid=0,recvseqid=0,readseqid=0 WHERE tenant_id=? AND topic=? AND userid=?", - sub.CreatedAt, sub.UpdatedAt, sub.ModeWant.String(), sub.ModeGiven.String(), tenantID, sub.Topic, decodedUID) + ), sub.CreatedAt, sub.UpdatedAt, sub.ModeWant.String(), sub.ModeGiven.String(), tenantID, sub.Topic, decodedUID) } else { - _, err = tx.Exec("UPDATE im_subscriptions SET createdat=?,updatedat=?,deletedat=NULL,modeWant=?,modeGiven=?,"+ + _, err = tx.Exec(tenantShardSQL(tenantID, "UPDATE im_subscriptions SET createdat=?,updatedat=?,deletedat=NULL,modeWant=?,modeGiven=?,"+ "delid=0,recvseqid=0,readseqid=0,private=? WHERE tenant_id=? AND topic=? AND userid=?", - sub.CreatedAt, sub.UpdatedAt, sub.ModeWant.String(), sub.ModeGiven.String(), jpriv, tenantID, sub.Topic, decodedUID) + ), sub.CreatedAt, sub.UpdatedAt, sub.ModeWant.String(), sub.ModeGiven.String(), jpriv, tenantID, sub.Topic, decodedUID) } } if err == nil && isOwner { - _, err = tx.Exec("UPDATE im_topics SET owner=? WHERE tenant_id=? AND name=?", decodedUID, tenantID, sub.Topic) + _, err = tx.Exec(tenantShardSQL(tenantID, "UPDATE im_topics SET owner=? WHERE tenant_id=? AND name=?"), decodedUID, tenantID, sub.Topic) } return err } @@ -1033,7 +1033,7 @@ func (a *adapter) TenantTopicCreateP2P(id t.TenantID, one, two *t.Subscription) topic.ObjHeader.MergeTimes(&one.ObjHeader) topic.TouchedAt = one.GetTouchedAt() var existingTopicID int64 - err = tx.Get(&existingTopicID, "SELECT id FROM im_topics WHERE tenant_id=? AND name=? ORDER BY id ASC LIMIT 1", id, topic.Id) + err = tx.Get(&existingTopicID, tenantShardSQL(id, "SELECT id FROM im_topics WHERE tenant_id=? AND name=? ORDER BY id ASC LIMIT 1"), id, topic.Id) if err != nil && err != sql.ErrNoRows { return err } @@ -1041,7 +1041,7 @@ func (a *adapter) TenantTopicCreateP2P(id t.TenantID, one, two *t.Subscription) if err = tenantTopicCreate(tx, topic); err != nil { return err } - } else if _, err = tx.ExecContext(ctx, "UPDATE im_topics SET updatedat=?,touchedat=?,state=? WHERE tenant_id=? AND id=?", + } else if _, err = tx.ExecContext(ctx, tenantShardSQL(id, "UPDATE im_topics SET updatedat=?,touchedat=?,state=? WHERE tenant_id=? AND id=?"), topic.UpdatedAt, topic.TouchedAt, t.StateOK, id, existingTopicID); err != nil { return err } @@ -1067,7 +1067,7 @@ func (a *adapter) TenantTopicGet(id t.TenantID, topic string) (*t.Topic, error) var top t.Topic err = a.db.GetContext(ctx, &top, "SELECT tenant_id,createdat,updatedat,state,stateat,touchedat,name AS id,usebt,access,owner,seqid,delid,subcnt,IFNULL(mutation_version,0) AS mutation_version,public,trusted,tags,aux "+ - "FROM im_topics WHERE tenant_id=? AND name=?", id, topic) + tenantShardSQL(id, "FROM im_topics WHERE tenant_id=? AND name=?"), id, topic) if err == sql.ErrNoRows { return nil, nil } @@ -1077,13 +1077,13 @@ func (a *adapter) TenantTopicGet(id t.TenantID, topic string) (*t.Topic, error) if t.GetTopicCat(topic) == t.TopicCatGrp { var subCnt int if err = a.db.GetContext(ctx, &subCnt, - "SELECT COUNT(*) FROM im_subscriptions WHERE tenant_id=? AND topic IN (?,?) AND deletedat IS NULL", + tenantShardSQL(id, "SELECT COUNT(*) FROM im_subscriptions WHERE tenant_id=? AND topic IN (?,?) AND deletedat IS NULL"), id, topic, t.GrpToChn(topic)); err != nil { return nil, err } if subCnt != top.SubCnt { top.SubCnt = subCnt - if _, err = a.db.ExecContext(ctx, "UPDATE im_topics SET subcnt=? WHERE tenant_id=? AND name=?", subCnt, id, topic); err != nil { + if _, err = a.db.ExecContext(ctx, tenantShardSQL(id, "UPDATE im_topics SET subcnt=? WHERE tenant_id=? AND name=?"), subCnt, id, topic); err != nil { return nil, err } } @@ -1106,6 +1106,7 @@ func (a *adapter) TenantTopicsForUser(id t.TenantID, uid t.Uid, keepDeleted bool } q := `SELECT tenant_id,createdat,updatedat,deletedat,topic,delid,recvseqid, readseqid,modewant,modegiven,private FROM im_subscriptions WHERE tenant_id=? AND userid=?` + q = tenantShardSQL(id, q) args := []any{id, decodeUID(uid)} if !keepDeleted { q += " AND deletedat IS NULL" @@ -1190,7 +1191,7 @@ func (a *adapter) TenantTopicsForUser(id t.TenantID, uid t.Uid, keepDeleted bool return nil, err } if len(topq) > 0 { - q = "SELECT tenant_id,updatedat,state,touchedat,name AS id,usebt,access,seqid,delid,subcnt,IFNULL(mutation_version,0) AS mutation_version,public,trusted FROM im_topics WHERE tenant_id=? AND name IN (?)" + q = tenantShardSQL(id, "SELECT tenant_id,updatedat,state,touchedat,name AS id,usebt,access,seqid,delid,subcnt,IFNULL(mutation_version,0) AS mutation_version,public,trusted FROM im_topics WHERE tenant_id=? AND name IN (?)") q, args, _ = sqlx.In(q, id, topq) if !keepDeleted { q += " AND state!=?" @@ -1306,6 +1307,7 @@ func (a *adapter) TenantUsersForTopic(id t.TenantID, topic string, keepDeleted b FROM im_subscriptions AS s JOIN im_users AS u ON s.tenant_id=u.tenant_id AND s.userid=u.id LEFT JOIN im_user_profiles AS p ON p.tenant_id=s.tenant_id AND p.user_id=s.userid WHERE s.tenant_id=? AND s.topic=?` + q = tenantShardSQL(id, q) args := []any{id, topic} if !keepDeleted { q += " AND u.state!=?" @@ -1426,7 +1428,7 @@ func (a *adapter) TenantOwnTopics(id t.TenantID, uid t.Uid) ([]string, error) { if a.tenantScopedDB { return a.OwnTopics(uid) } - return a.tenantTopicNames("SELECT name FROM im_topics WHERE tenant_id=? AND owner=? AND state!=?", + return a.tenantTopicNames(tenantShardSQL(id, "SELECT name FROM im_topics WHERE tenant_id=? AND owner=? AND state!=?"), id, decodeUID(uid), t.StateDeleted) } @@ -1441,8 +1443,8 @@ func (a *adapter) TenantChannelsForUser(id t.TenantID, uid t.Uid) ([]string, err if a.tenantScopedDB { return a.ChannelsForUser(uid) } - return a.tenantTopicNames("SELECT topic FROM im_subscriptions WHERE tenant_id=? AND userid=? AND topic LIKE 'chn%' "+ - "AND INSTR(modewant,'P')>0 AND INSTR(modegiven,'P')>0 AND deletedat IS NULL", id, decodeUID(uid)) + return a.tenantTopicNames(tenantShardSQL(id, "SELECT topic FROM im_subscriptions WHERE tenant_id=? AND userid=? AND topic LIKE 'chn%' "+ + "AND INSTR(modewant,'P')>0 AND INSTR(modegiven,'P')>0 AND deletedat IS NULL"), id, decodeUID(uid)) } func (a *adapter) TenantTopicShare(id t.TenantID, topic string, subs []*t.Subscription) error { @@ -1480,7 +1482,7 @@ func (a *adapter) TenantTopicShare(id t.TenantID, topic string, subs []*t.Subscr } } if topic != "" { - if _, err = tx.Exec("UPDATE im_topics SET subcnt=subcnt+? WHERE tenant_id=? AND name=?", len(subs), id, topic); err != nil { + if _, err = tx.Exec(tenantShardSQL(id, "UPDATE im_topics SET subcnt=subcnt+? WHERE tenant_id=? AND name=?"), len(subs), id, topic); err != nil { return err } } @@ -1563,11 +1565,7 @@ func groupMessageDeleteList(tx *sqlx.Tx, tenantID t.TenantID, topic string, toDe return err } } - if _, err = tx.Exec("DELETE FROM im_group_message_topic_shards WHERE tenant_id=? AND topic=?", tenantID, topic); err != nil { - return err - } - _, err = tx.Exec("DELETE FROM im_group_message_dedup WHERE tenant_id=? AND topic=?", tenantID, topic) - return err + return nil } delRanges := toDel.SeqIdRanges if toDel.DeletedFor == "" { @@ -1659,7 +1657,7 @@ func (a *adapter) TenantTopicDelete(id t.TenantID, topic string, isChan, hard bo } }() if hard { - q, args, _ := sqlx.In("DELETE FROM im_subscriptions WHERE tenant_id=? AND topic IN (?)", id, topics) + q, args, _ := sqlx.In(tenantShardSQL(id, "DELETE FROM im_subscriptions WHERE tenant_id=? AND topic IN (?)"), id, topics) if _, err = tx.Exec(tx.Rebind(q), args...); err != nil { return err } @@ -1674,16 +1672,16 @@ func (a *adapter) TenantTopicDelete(id t.TenantID, topic string, isChan, hard bo if _, err = tx.Exec("DELETE FROM im_topictags WHERE tenant_id=? AND topic=?", id, topic); err != nil { return err } - if _, err = tx.Exec("DELETE FROM im_topics WHERE tenant_id=? AND name=?", id, topic); err != nil { + if _, err = tx.Exec(tenantShardSQL(id, "DELETE FROM im_topics WHERE tenant_id=? AND name=?"), id, topic); err != nil { return err } } else { now := t.TimeNow() - q, args, _ := sqlx.In("UPDATE im_subscriptions SET updatedat=?,deletedat=? WHERE tenant_id=? AND topic IN (?)", now, now, id, topics) + q, args, _ := sqlx.In(tenantShardSQL(id, "UPDATE im_subscriptions SET updatedat=?,deletedat=? WHERE tenant_id=? AND topic IN (?)"), now, now, id, topics) if _, err = tx.Exec(tx.Rebind(q), args...); err != nil { return err } - if _, err = tx.Exec("UPDATE im_topics SET updatedat=?,touchedat=?,state=?,stateat=? WHERE tenant_id=? AND name=?", + if _, err = tx.Exec(tenantShardSQL(id, "UPDATE im_topics SET updatedat=?,touchedat=?,state=?,stateat=? WHERE tenant_id=? AND name=?"), now, now, t.StateDeleted, now, id, topic); err != nil { return err } @@ -1706,7 +1704,7 @@ func (a *adapter) TenantTopicUpdateOnMessage(id t.TenantID, topic string, msg *t if cancel != nil { defer cancel() } - _, err = a.db.ExecContext(ctx, "UPDATE im_topics SET seqid=?,touchedat=? WHERE tenant_id=? AND name=?", + _, err = a.db.ExecContext(ctx, tenantShardSQL(id, "UPDATE im_topics SET seqid=?,touchedat=? WHERE tenant_id=? AND name=?"), msg.SeqId, msg.CreatedAt, id, topic) return err } @@ -1727,7 +1725,7 @@ func (a *adapter) TenantTopicUpdateSubCnt(id t.TenantID, topic string) error { defer cancel() } _, err = a.db.ExecContext(ctx, - "UPDATE im_topics SET subcnt=(SELECT COUNT(*) FROM im_subscriptions WHERE tenant_id=? AND topic IN (?,?) AND deletedat IS NULL) WHERE tenant_id=? AND name=?", + tenantShardSQL(id, "UPDATE im_topics SET subcnt=(SELECT COUNT(*) FROM im_subscriptions WHERE tenant_id=? AND topic IN (?,?) AND deletedat IS NULL) WHERE tenant_id=? AND name=?"), id, topic, t.GrpToChn(topic), id, topic) return err } @@ -1761,7 +1759,7 @@ func (a *adapter) TenantTopicUpdate(id t.TenantID, topic string, update map[stri } cols, args := common.UpdateByMap(update) args = append(args, id, topic) - res, err := tx.Exec("UPDATE im_topics SET "+strings.Join(cols, ",")+" WHERE tenant_id=? AND name=?", args...) + res, err := tx.Exec(tenantShardSQL(id, "UPDATE im_topics SET "+strings.Join(cols, ",")+" WHERE tenant_id=? AND name=?"), args...) if err != nil { return err } @@ -1813,6 +1811,7 @@ func (a *adapter) TenantTopicBumpMutationVersion(id t.TenantID, topic string, at query = "SELECT mutation_version FROM im_topics WHERE tenant_id=? AND name=?" args = []any{id, topic} } + query = tenantShardSQL(id, query) if err = tx.GetContext(ctx, &version, query, args...); err != nil { return 0, err } @@ -1837,7 +1836,7 @@ func (a *adapter) TenantTopicOwnerChange(id t.TenantID, topic string, owner t.Ui if cancel != nil { defer cancel() } - _, err = a.db.ExecContext(ctx, "UPDATE im_topics SET owner=? WHERE tenant_id=? AND name=?", decodeUID(owner), id, topic) + _, err = a.db.ExecContext(ctx, tenantShardSQL(id, "UPDATE im_topics SET owner=? WHERE tenant_id=? AND name=?"), decodeUID(owner), id, topic) return err } @@ -1858,6 +1857,7 @@ func (a *adapter) TenantSubscriptionGet(id t.TenantID, topic string, uid t.Uid, } query := `SELECT tenant_id,createdat,updatedat,deletedat,userid AS user,topic,delid,recvseqid, readseqid,modewant,modegiven,private FROM im_subscriptions WHERE tenant_id=? AND topic=? AND userid=?` + query = tenantShardSQL(id, query) if !keepDeleted { query += " AND deletedat IS NULL" } @@ -1890,8 +1890,8 @@ func (a *adapter) TenantSubsForUser(id t.TenantID, uid t.Uid) ([]t.Subscription, subs, err := a.SubsForUser(uid) return stampTenantSubs(id, subs), err } - rows, err := a.db.Queryx(`SELECT tenant_id,createdat,updatedat,deletedat,userid AS user,topic,delid,recvseqid, - readseqid,modewant,modegiven FROM im_subscriptions WHERE tenant_id=? AND userid=? AND deletedat IS NULL`, + rows, err := a.db.Queryx(tenantShardSQL(id, `SELECT tenant_id,createdat,updatedat,deletedat,userid AS user,topic,delid,recvseqid, + readseqid,modewant,modegiven FROM im_subscriptions WHERE tenant_id=? AND userid=? AND deletedat IS NULL`), id, decodeUID(uid)) if err != nil { return nil, err @@ -1923,6 +1923,7 @@ func (a *adapter) TenantSubsForTopic(id t.TenantID, topic string, keepDeleted bo } q := `SELECT tenant_id,createdat,updatedat,deletedat,userid AS user,topic,delid,recvseqid, readseqid,modewant,modegiven,private FROM im_subscriptions WHERE tenant_id=? AND topic=?` + q = tenantShardSQL(id, q) args := []any{id, topic} if !keepDeleted { q += " AND deletedat IS NULL" @@ -1976,7 +1977,7 @@ func (a *adapter) TenantSubsUpdate(id t.TenantID, topic string, uid t.Uid, updat return a.SubsUpdate(topic, uid, update) } cols, args := common.UpdateByMap(update) - q := "UPDATE im_subscriptions SET " + strings.Join(cols, ",") + " WHERE tenant_id=? AND topic=?" + q := tenantShardSQL(id, "UPDATE im_subscriptions SET ") + strings.Join(cols, ",") + " WHERE tenant_id=? AND topic=?" args = append(args, id, topic) if !uid.IsZero() { q += " AND userid=?" @@ -2016,7 +2017,7 @@ func (a *adapter) TenantSubsDelete(id t.TenantID, topic string, uid t.Uid) error }() now := t.TimeNow() decoded := decodeUID(uid) - res, err := tx.Exec("UPDATE im_subscriptions SET updatedat=?,deletedat=? WHERE tenant_id=? AND topic=? AND userid=? AND deletedat IS NULL", + res, err := tx.Exec(tenantShardSQL(id, "UPDATE im_subscriptions SET updatedat=?,deletedat=? WHERE tenant_id=? AND topic=? AND userid=? AND deletedat IS NULL"), now, now, id, topic, decoded) if err != nil { return err @@ -2030,7 +2031,7 @@ func (a *adapter) TenantSubsDelete(id t.TenantID, topic string, uid t.Uid) error } } if t.GetTopicCat(topic) == t.TopicCatGrp { - if _, err = tx.Exec("UPDATE im_topics SET subcnt=subcnt-1 WHERE tenant_id=? AND name=?", id, t.ChnToGrp(topic)); err != nil { + if _, err = tx.Exec(tenantShardSQL(id, "UPDATE im_topics SET subcnt=subcnt-1 WHERE tenant_id=? AND name=?"), id, t.ChnToGrp(topic)); err != nil { return err } } @@ -2038,7 +2039,7 @@ func (a *adapter) TenantSubsDelete(id t.TenantID, topic string, uid t.Uid) error } func tenantSubsDelForUser(tx *sqlx.Tx, tenantID t.TenantID, decodedUID int64, hard bool) error { - rows, err := tx.Query("SELECT topic FROM im_subscriptions WHERE tenant_id=? AND userid=? AND deletedat IS NULL", tenantID, decodedUID) + rows, err := tx.Query(tenantShardSQL(tenantID, "SELECT topic FROM im_subscriptions WHERE tenant_id=? AND userid=? AND deletedat IS NULL"), tenantID, decodedUID) if err != nil { return err } @@ -2060,16 +2061,16 @@ func tenantSubsDelForUser(tx *sqlx.Tx, tenantID t.TenantID, decodedUID int64, ha return err } if len(topics) > 0 { - q, args, _ := sqlx.In("UPDATE im_topics SET subcnt=subcnt-1 WHERE tenant_id=? AND name IN (?)", tenantID, topics) + q, args, _ := sqlx.In(tenantShardSQL(tenantID, "UPDATE im_topics SET subcnt=subcnt-1 WHERE tenant_id=? AND name IN (?)"), tenantID, topics) if _, err = tx.Exec(tx.Rebind(q), args...); err != nil { return err } } if hard { - _, err = tx.Exec("DELETE FROM im_subscriptions WHERE tenant_id=? AND userid=?", tenantID, decodedUID) + _, err = tx.Exec(tenantShardSQL(tenantID, "DELETE FROM im_subscriptions WHERE tenant_id=? AND userid=?"), tenantID, decodedUID) } else { now := t.TimeNow() - _, err = tx.Exec("UPDATE im_subscriptions SET updatedat=?,deletedat=? WHERE tenant_id=? AND userid=? AND deletedat IS NULL", + _, err = tx.Exec(tenantShardSQL(tenantID, "UPDATE im_subscriptions SET updatedat=?,deletedat=? WHERE tenant_id=? AND userid=? AND deletedat IS NULL"), now, now, tenantID, decodedUID) } return err @@ -2111,6 +2112,7 @@ func (a *adapter) TenantFind(id t.TenantID, caller, promoPrefix string, req [][] "FROM im_topics AS t JOIN im_topictags AS tg ON t.tenant_id=tg.tenant_id AND t.name=tg.topic " + "WHERE t.tenant_id=? AND " + stateConstraint + "tg.tag IN (?" + strings.Repeat(",?", len(allTags)-1) + ") " + "GROUP BY t.name,t.createdat,t.updatedat,t.usebt,t.access,t.subcnt,t.public,t.trusted,t.tags " + query = tenantShardSQL(id, query) if len(allReq) > 0 { q, addArgs := common.DisjunctionSql(req, "tg.tag") query += q @@ -2177,8 +2179,8 @@ func (a *adapter) TenantFindOne(id t.TenantID, tag string) (string, error) { defer cancel() } rows, err := a.db.QueryxContext(ctx, - "SELECT tp.name AS topic FROM im_topics AS tp LEFT JOIN im_topictags AS tt ON tp.tenant_id=tt.tenant_id AND tp.name=tt.topic "+ - "WHERE tp.tenant_id=? AND tt.tag=? LIMIT 1", id, tag) + tenantShardSQL(id, "SELECT tp.name AS topic FROM im_topics AS tp LEFT JOIN im_topictags AS tt ON tp.tenant_id=tt.tenant_id AND tp.name=tt.topic "+ + "WHERE tp.tenant_id=? AND tt.tag=? LIMIT 1"), id, tag) if err != nil { return "", err } @@ -2273,17 +2275,6 @@ func (a *adapter) TenantGroupMessageSave(id t.TenantID, msg *t.Message) error { } defer tx.Rollback() messageID := store.DecodeUid(msg.Uid()) - if clientMsgID.Valid { - if _, err = tx.ExecContext(ctx, `INSERT INTO im_group_message_dedup - (tenant_id,topic,from_user,client_msg_id,seqid,shard_month,message_id,status,expires_at) - VALUES(?,?,?,?,?,?,?,'done',?)`, id, msg.Topic, decodeUID(t.ParseUid(msg.From)), clientMsgID, - msg.SeqId, msg.ShardMonth, messageID, time.Now().UTC().Add(30*24*time.Hour)); err != nil { - if isDupe(err) { - return t.ErrDuplicate - } - return err - } - } query := fmt.Sprintf(`INSERT INTO %s (tenant_id,message_id,topic,group_id,seqid,from_user,client_msg_id,msg_type,status,content,head,mentioned_user_ids,mention_all, file_count,content_size,version,created_at,updated_at,content_nonce,key_version) @@ -2298,21 +2289,6 @@ func (a *adapter) TenantGroupMessageSave(id t.TenantID, msg *t.Message) error { if err != nil { return err } - if _, err = tx.ExecContext(ctx, `INSERT INTO im_group_message_topic_shards - (tenant_id,topic,shard_month,min_seq,max_seq,message_count,min_created_at,max_created_at) - VALUES(?,?,?,?,?,1,?,?) ON DUPLICATE KEY UPDATE - min_seq=LEAST(min_seq,VALUES(min_seq)),max_seq=GREATEST(max_seq,VALUES(max_seq)), - message_count=message_count+1,min_created_at=LEAST(min_created_at,VALUES(min_created_at)), - max_created_at=GREATEST(max_created_at,VALUES(max_created_at))`, - id, msg.Topic, msg.ShardMonth, msg.SeqId, msg.SeqId, msg.CreatedAt, msg.CreatedAt); err != nil { - return err - } - if _, err = tx.ExecContext(ctx, `UPDATE im_group_message_shards SET row_count=row_count+1, - min_created_at=IF(min_created_at IS NULL,?,LEAST(min_created_at,?)), - max_created_at=IF(max_created_at IS NULL,?,GREATEST(max_created_at,?)) WHERE tenant_id=? AND shard_month=?`, - msg.CreatedAt, msg.CreatedAt, msg.CreatedAt, msg.CreatedAt, id, msg.ShardMonth); err != nil { - return err - } return tx.Commit() } @@ -2366,19 +2342,6 @@ func (a *adapter) TenantGroupMessageGetByClientID(id t.TenantID, topic string, f if cancel != nil { defer cancel() } - var seq int64 - if err = a.db.QueryRowContext(ctx, `SELECT seqid FROM im_group_message_dedup - WHERE tenant_id=? AND topic=? AND from_user=? AND client_msg_id=?`, id, topic, decodeUID(from), clientMsgID).Scan(&seq); err == nil { - msg, getErr := tenantGroupMessageGetOne(a.db, ctx, id, topic, seq, "") - if getErr == nil { - return msg, nil - } - if getErr != sql.ErrNoRows { - return nil, getErr - } - } else if err != sql.ErrNoRows { - return nil, err - } tables, err := groupMessageTablesForTopic(a.db, ctx, id, topic) if err != nil { return nil, err @@ -3111,14 +3074,14 @@ func (a *adapter) TenantMessageMutations(id t.TenantID, topic string, uid t.Uid, } var currentVersion int64 if a.tenantScopedDB { - if err := a.db.GetContext(ctx, ¤tVersion, "SELECT IFNULL(mutation_version,0) FROM im_topics WHERE name=?", topic); err != nil { + if err := a.db.GetContext(ctx, ¤tVersion, tenantShardSQL(id, "SELECT IFNULL(mutation_version,0) FROM im_topics WHERE name=?"), topic); err != nil { if err == sql.ErrNoRows { return nil, "", false, 0, t.ErrNotFound } return nil, "", false, 0, err } } else { - if err := a.db.GetContext(ctx, ¤tVersion, "SELECT IFNULL(mutation_version,0) FROM im_topics WHERE tenant_id=? AND name=?", id, topic); err != nil { + if err := a.db.GetContext(ctx, ¤tVersion, tenantShardSQL(id, "SELECT IFNULL(mutation_version,0) FROM im_topics WHERE tenant_id=? AND name=?"), id, topic); err != nil { if err == sql.ErrNoRows { return nil, "", false, 0, t.ErrNotFound } @@ -5261,7 +5224,7 @@ func (a *adapter) TenantFileUserHasAccess(id t.TenantID, fid string, uid t.Uid) defer cancel() } var allowed int - err = a.db.GetContext(ctx, &allowed, fmt.Sprintf(` + err = a.db.GetContext(ctx, &allowed, tenantShardSQL(id, fmt.Sprintf(` SELECT COUNT(*) FROM im_fileuploads AS fu WHERE fu.tenant_id=? AND (fu.id=? OR fu.location=? OR fu.location LIKE ?) AND fu.status=? AND ( fu.userid=? @@ -5310,7 +5273,7 @@ func (a *adapter) TenantFileUserHasAccess(id t.TenantID, fid string, uid t.Uid) WHERE (gf.file_id=? OR gf.file_id LIKE CONCAT(?,'.%%')) AND gf.status='active' AND s.deletedat IS NULL AND INSTR(s.modewant,'R')>0 AND INSTR(s.modegiven,'R')>0 ) - ) LIMIT 1`, tenantFriendshipsTable(id), tenantMessagesTable(id)), + ) LIMIT 1`, tenantFriendshipsTable(id), tenantMessagesTable(id))), id, decodeUID(fileID), fileID.String32(), "%/"+fileID.String32(), t.UploadCompleted, decodeUID(uid), decodeUID(uid), decodeUID(uid), t.RelationshipFriend, decodeUID(uid), decodeUID(uid), t.GroupMemberStatusActive, decodeUID(uid), decodeUID(uid), decodeUID(uid), fid, fid) @@ -5379,33 +5342,14 @@ func (a *adapter) TenantVerificationCreate(id t.TenantID, vc *t.VerificationCode return t.ErrMalformed } - ctx, cancel := a.getContextForTx() + ctx, cancel := a.getContext() if cancel != nil { defer cancel() } - tx, err := a.db.BeginTxx(ctx, nil) - if err != nil { - return err - } - defer func() { - if err != nil { - tx.Rollback() - } - }() - now := t.TimeNow() - _, err = tx.ExecContext(ctx, - `UPDATE im_verification_codes - SET state=?,updated_at=? - WHERE tenant_id=? AND scene=? AND phone_hash=? AND state=?`, - t.VerificationStateExpired, now, id, vc.Scene, vc.PhoneHash, t.VerificationStatePending) - if err != nil { - return err - } - vc.CreatedAt = now vc.UpdatedAt = now - _, err = tx.ExecContext(ctx, + _, err := a.db.ExecContext(ctx, `INSERT INTO im_verification_codes (tenant_id,challenge_id,scene,action,phone_hash,phone_mask,target_user,code_hash,state,attempts,max_attempts, expires_at,resend_after,created_at,updated_at) @@ -5418,7 +5362,7 @@ func (a *adapter) TenantVerificationCreate(id t.TenantID, vc *t.VerificationCode if err != nil { return err } - return tx.Commit() + return nil } func (a *adapter) TenantVerificationGet(id t.TenantID, challengeID string) (*t.VerificationCode, error) { @@ -5854,23 +5798,10 @@ func (a *adapter) TenantActionTokenCreate(id t.TenantID, token *t.ActionToken) e token.TokenID == "" || token.Action == "" || token.TokenHash == "" || token.ExpiresAt.IsZero() { return t.ErrMalformed } - ctx, cancel := a.getContext() - if cancel != nil { - defer cancel() - } now := t.TimeNow() token.CreatedAt = now token.UpdatedAt = now - _, err := a.db.ExecContext(ctx, - `INSERT INTO im_action_tokens - (tenant_id,user_id,token_id,action,target,token_hash,state,expires_at,created_at,updated_at) - VALUES(?,?,?,?,?,?,?,?,?,?)`, - id, decodeUID(token.UserID), token.TokenID, token.Action, token.Target, token.TokenHash, token.State, - token.ExpiresAt, token.CreatedAt, token.UpdatedAt) - if isDupe(err) { - return t.ErrDuplicate - } - return err + return nil } func (a *adapter) TenantActionTokenConsume(id t.TenantID, uid t.Uid, token, action, target string, now time.Time) (*t.ActionToken, error) { @@ -5893,50 +5824,7 @@ func (a *adapter) TenantActionTokenConsume(id t.TenantID, uid t.Uid, token, acti if !ok || tokenID == "" || secret == "" { return nil, t.ErrMalformed } - tokenHash := securityTokenHash(id, uid, action, target, tokenID, secret) - ctx, cancel := a.getContextForTx() - if cancel != nil { - defer cancel() - } - tx, err := a.db.BeginTxx(ctx, nil) - if err != nil { - return nil, err - } - defer func() { - if err != nil { - tx.Rollback() - } - }() - var at t.ActionToken - err = tx.GetContext(ctx, &at, - `SELECT id,user_id,token_id,action,target,token_hash,state,expires_at,consumed_at,created_at,updated_at - FROM im_action_tokens - WHERE tenant_id=? AND user_id=? AND token_id=? AND action=? AND target=? LIMIT 1`, - id, decodeUID(uid), tokenID, action, target) - if err == sql.ErrNoRows { - return nil, t.ErrNotFound - } - if err != nil { - return nil, err - } - if at.State != 0 || at.TokenHash != tokenHash || !now.Before(at.ExpiresAt) { - return nil, t.ErrNotFound - } - res, err := tx.ExecContext(ctx, - `UPDATE im_action_tokens SET state=1,consumed_at=?,updated_at=? - WHERE tenant_id=? AND user_id=? AND token_id=? AND action=? AND target=? AND state=0 AND expires_at>?`, - now, now, id, decodeUID(uid), tokenID, action, target, now) - if err != nil { - return nil, err - } - if rows, _ := res.RowsAffected(); rows == 0 { - return nil, t.ErrNotFound - } - if err = tx.Commit(); err != nil { - return nil, err - } - at.TenantID = id - return &at, nil + return nil, t.ErrUnsupported } func (a *adapter) TenantIndependentPasswordGet(id t.TenantID, uid t.Uid, purpose string) (*t.IndependentPassword, error) { @@ -6166,7 +6054,7 @@ func (a *adapter) TenantSystemNotificationCreate(id t.TenantID, notification *t. notification.CreatedAt = now notification.UpdatedAt = now _, err := a.db.ExecContext(ctx, - `INSERT INTO im_system_notifications + tenantShardSQL(id, `INSERT INTO im_system_notifications (tenant_id,user_id,event_id,event_type,title,summary,action,read_at,created_at,updated_at) VALUES(?,?,?,?,?,?,?,?,?,?) ON DUPLICATE KEY UPDATE @@ -6174,7 +6062,7 @@ func (a *adapter) TenantSystemNotificationCreate(id t.TenantID, notification *t. title=VALUES(title), summary=VALUES(summary), action=VALUES(action), - updated_at=VALUES(updated_at)`, + updated_at=VALUES(updated_at)`), id, decodeUID(notification.UserID), notification.EventID, notification.EventType, notification.Title, notification.Summary, common.ToJSON(notification.Action), notification.ReadAt, notification.CreatedAt, notification.UpdatedAt) return err @@ -6207,6 +6095,7 @@ func (a *adapter) TenantSystemNotificationList(id t.TenantID, uid t.Uid, cursor query := `SELECT id,user_id,event_id,event_type,title,summary,action,read_at,created_at,updated_at FROM im_system_notifications WHERE tenant_id=? AND user_id=?` + query = tenantShardSQL(id, query) args := []any{id, decodeUID(uid)} if lastID > 0 { query += " AND id 0 { where += " AND version=?" args = append(args, patch.Version) @@ -437,7 +441,7 @@ func (a *adapter) TenantGroupMemberUpdate(id t.TenantID, topic string, uid t.Uid return nil, err } defer tx.Rollback() - res, err := tx.ExecContext(ctx, "UPDATE im_group_members SET "+strings.Join(set, ",")+" WHERE "+where, args...) + res, err := tx.ExecContext(ctx, tenantShardSQL(id, "UPDATE im_group_members SET "+strings.Join(set, ",")+" WHERE "+where), args...) if err != nil { return nil, err } @@ -447,13 +451,13 @@ func (a *adapter) TenantGroupMemberUpdate(id t.TenantID, topic string, uid t.Uid } return nil, t.ErrNotFound } - if err = recountGroupMembersTx(tx, topic); err != nil { + if err = recountGroupMembersTx(tx, id, topic); err != nil { return nil, err } if err = tx.Commit(); err != nil { return nil, err } - return a.groupMemberByQuery(ctx, "SELECT * FROM im_group_members WHERE topic=? AND user_id=?", topic, decodeUID(uid)) + return a.groupMemberByQuery(ctx, tenantShardSQL(id, "SELECT * FROM im_group_members WHERE tenant_id=? AND topic=? AND user_id=?"), id, topic, decodeUID(uid)) } func (a *adapter) TenantGroupInvitationCreate(id t.TenantID, inv *t.GroupInvitation) (*t.GroupInvitation, error) { @@ -572,8 +576,8 @@ func (a *adapter) TenantGroupInvitationAccept(id t.TenantID, invitationID string return &inv, nil } var deletedAt sql.NullTime - err = tx.QueryRowxContext(ctx, "SELECT deletedat FROM im_subscriptions WHERE topic=? AND userid=?", - inv.Topic, decodeUID(uid)).Scan(&deletedAt) + err = tx.QueryRowxContext(ctx, tenantShardSQL(id, "SELECT deletedat FROM im_subscriptions WHERE tenant_id=? AND topic=? AND userid=?"), + id, inv.Topic, decodeUID(uid)).Scan(&deletedAt) if err != nil && err != sql.ErrNoRows { return nil, err } @@ -581,20 +585,20 @@ func (a *adapter) TenantGroupInvitationAccept(id t.TenantID, invitationID string sub := &t.Subscription{TenantID: id, User: uid.String(), Topic: inv.Topic, ModeWant: t.ModeCPublic, ModeGiven: t.ModeCPublic, Private: memberPrivate} sub.InitTimes() - if err = createSubscription(tx, sub, deletedAt.Valid); err != nil { + if err = createTenantSubscription(tx, id, sub, deletedAt.Valid); err != nil { return nil, err } } member := &t.GroupMember{Topic: inv.Topic, UserID: uid, Role: t.GroupMemberRoleMember, MemberStatus: t.GroupMemberStatusActive, JoinSource: "invitation", JoinedBy: inv.InviterUserID, JoinedAt: now, Version: 1, CreatedAt: now, UpdatedAt: now} - if err = upsertGroupMemberTx(tx, member); err != nil { + if err = upsertGroupMemberTx(tx, id, member); err != nil { return nil, err } - if err = recountGroupMembersAndBumpVersionTx(tx, inv.Topic); err != nil { + if err = recountGroupMembersAndBumpVersionTx(tx, id, inv.Topic); err != nil { return nil, err } - if err = updateGroupTopicSubCountTx(tx, inv.Topic); err != nil { + if err = updateGroupTopicSubCountTx(tx, id, inv.Topic); err != nil { return nil, err } if _, err = tx.ExecContext(ctx, `UPDATE im_group_invitations @@ -761,13 +765,13 @@ func (a *adapter) TenantGroupJoinRequestResolve(id t.TenantID, topic, requestID sub := &t.Subscription{TenantID: id, User: current.ApplicantUserID.String(), Topic: topic, ModeWant: t.ModeCPublic, ModeGiven: t.ModeCPublic, Private: memberPrivate} sub.InitTimes() - if err = createSubscription(tx, sub, false); err != nil { + if err = createTenantSubscription(tx, id, sub, false); err != nil { return nil, err } member := &t.GroupMember{Topic: topic, UserID: current.ApplicantUserID, Role: t.GroupMemberRoleMember, MemberStatus: t.GroupMemberStatusActive, JoinSource: "approval", JoinedBy: actor, JoinedAt: now, Version: 1, CreatedAt: now, UpdatedAt: now} - if err = upsertGroupMemberTx(tx, member); err != nil { + if err = upsertGroupMemberTx(tx, id, member); err != nil { return nil, err } } @@ -776,10 +780,10 @@ func (a *adapter) TenantGroupJoinRequestResolve(id t.TenantID, topic, requestID return nil, err } if action == "accept" { - if err = recountGroupMembersAndBumpVersionTx(tx, topic); err != nil { + if err = recountGroupMembersAndBumpVersionTx(tx, id, topic); err != nil { return nil, err } - if err = updateGroupTopicSubCountTx(tx, topic); err != nil { + if err = updateGroupTopicSubCountTx(tx, id, topic); err != nil { return nil, err } } @@ -799,7 +803,7 @@ func (a *adapter) TenantGroupAnnouncementLatest(id t.TenantID, topic string) (*t if cancel != nil { defer cancel() } - return a.groupAnnouncementByQuery(ctx, "SELECT "+groupAnnouncementSelectColumns+" FROM im_group_announcements WHERE topic=? AND status=? ORDER BY version DESC,id DESC LIMIT 1", topic, t.GroupAnnouncementStatusCurrent) + return a.groupAnnouncementByQuery(ctx, "SELECT "+groupAnnouncementSelectColumns(id)+" FROM im_group_announcements WHERE topic=? AND status=? ORDER BY version DESC,id DESC LIMIT 1", topic, t.GroupAnnouncementStatusCurrent) } func (a *adapter) TenantGroupAnnouncementGet(id t.TenantID, topic, announcementID string) (*t.GroupAnnouncement, error) { @@ -812,7 +816,7 @@ func (a *adapter) TenantGroupAnnouncementGet(id t.TenantID, topic, announcementI if cancel != nil { defer cancel() } - return a.groupAnnouncementByQuery(ctx, "SELECT "+groupAnnouncementSelectColumns+" FROM im_group_announcements WHERE topic=? AND announcement_id=? LIMIT 1", topic, announcementID) + return a.groupAnnouncementByQuery(ctx, "SELECT "+groupAnnouncementSelectColumns(id)+" FROM im_group_announcements WHERE topic=? AND announcement_id=? LIMIT 1", topic, announcementID) } func (a *adapter) TenantGroupAnnouncementPut(id t.TenantID, ann *t.GroupAnnouncement, expectedVersion int) (*t.GroupAnnouncement, error) { @@ -1233,7 +1237,7 @@ func (a *adapter) TenantGroupCreate(id t.TenantID, topic *t.Topic, owner t.Uid, ownerSub := &t.Subscription{TenantID: id, ObjHeader: t.ObjHeader{CreatedAt: topic.CreatedAt}, User: owner.String(), Topic: topic.Id, ModeGiven: t.ModeCFull, ModeWant: topic.GetAccess(owner), Private: ownerPrivate} ownerSub.InitTimes() - if err = createSubscription(tx, ownerSub, false); err != nil { + if err = createTenantSubscription(tx, id, ownerSub, false); err != nil { return err } settings.MemberCount = 1 @@ -1243,7 +1247,7 @@ func (a *adapter) TenantGroupCreate(id t.TenantID, topic *t.Topic, owner t.Uid, ownerMember := &t.GroupMember{Topic: topic.Id, UserID: owner, Role: t.GroupMemberRoleOwner, MemberStatus: t.GroupMemberStatusActive, JoinSource: "create", JoinedBy: owner, JoinedAt: now, Version: 1, CreatedAt: now, UpdatedAt: now} - if err = upsertGroupMemberTx(tx, ownerMember); err != nil { + if err = upsertGroupMemberTx(tx, id, ownerMember); err != nil { return err } seen := map[t.Uid]bool{owner: true} @@ -1255,20 +1259,20 @@ func (a *adapter) TenantGroupCreate(id t.TenantID, topic *t.Topic, owner t.Uid, sub := &t.Subscription{TenantID: id, User: uid.String(), Topic: topic.Id, ModeWant: t.ModeCPublic, ModeGiven: t.ModeCPublic, Private: memberPrivate} sub.InitTimes() - if err = createSubscription(tx, sub, false); err != nil { + if err = createTenantSubscription(tx, id, sub, false); err != nil { return err } member := &t.GroupMember{Topic: topic.Id, UserID: uid, Role: t.GroupMemberRoleMember, MemberStatus: t.GroupMemberStatusActive, JoinSource: "create", JoinedBy: owner, JoinedAt: now, Version: 1, CreatedAt: now, UpdatedAt: now} - if err = upsertGroupMemberTx(tx, member); err != nil { + if err = upsertGroupMemberTx(tx, id, member); err != nil { return err } } - if err = recountGroupMembersTx(tx, topic.Id); err != nil { + if err = recountGroupMembersTx(tx, id, topic.Id); err != nil { return err } - if err = updateGroupTopicSubCountTx(tx, topic.Id); err != nil { + if err = updateGroupTopicSubCountTx(tx, id, topic.Id); err != nil { return err } return tx.Commit() @@ -1291,7 +1295,7 @@ func (a *adapter) TenantGroupInviteMember(id t.TenantID, topic string, inviter, } defer tx.Rollback() var deletedAt sql.NullTime - err = tx.QueryRowContext(ctx, "SELECT deletedat FROM im_subscriptions WHERE topic=? AND userid=? FOR UPDATE", topic, decodeUID(target)).Scan(&deletedAt) + err = tx.QueryRowContext(ctx, tenantShardSQL(id, "SELECT deletedat FROM im_subscriptions WHERE tenant_id=? AND topic=? AND userid=? FOR UPDATE"), id, topic, decodeUID(target)).Scan(&deletedAt) if err != nil && err != sql.ErrNoRows { return false, err } @@ -1301,7 +1305,7 @@ func (a *adapter) TenantGroupInviteMember(id t.TenantID, topic string, inviter, sub := &t.Subscription{TenantID: id, User: target.String(), Topic: topic, ModeWant: t.ModeCPublic, ModeGiven: t.ModeCPublic, Private: private} sub.InitTimes() - if err = createSubscription(tx, sub, false); err != nil { + if err = createTenantSubscription(tx, id, sub, false); err != nil { return false, err } invitationID := "ginv_" + store.Store.GetUidString() @@ -1319,13 +1323,13 @@ func (a *adapter) TenantGroupInviteMember(id t.TenantID, topic string, inviter, member := &t.GroupMember{Topic: topic, UserID: target, Role: t.GroupMemberRoleMember, MemberStatus: t.GroupMemberStatusActive, JoinSource: "invitation", JoinedBy: inviter, JoinedAt: now, Version: 1, CreatedAt: now, UpdatedAt: now} - if err = upsertGroupMemberTx(tx, member); err != nil { + if err = upsertGroupMemberTx(tx, id, member); err != nil { return false, err } - if err = recountGroupMembersAndBumpVersionTx(tx, topic); err != nil { + if err = recountGroupMembersAndBumpVersionTx(tx, id, topic); err != nil { return false, err } - if err = updateGroupTopicSubCountTx(tx, topic); err != nil { + if err = updateGroupTopicSubCountTx(tx, id, topic); err != nil { return false, err } if err = tx.Commit(); err != nil { @@ -1349,21 +1353,21 @@ func (a *adapter) TenantGroupLeave(id t.TenantID, topic string, uid t.Uid, now t return err } defer tx.Rollback() - res, err := tx.ExecContext(ctx, "UPDATE im_subscriptions SET updatedat=?,deletedat=? WHERE topic=? AND userid=? AND deletedat IS NULL", now, now, topic, decodeUID(uid)) + res, err := tx.ExecContext(ctx, tenantShardSQL(id, "UPDATE im_subscriptions SET updatedat=?,deletedat=? WHERE tenant_id=? AND topic=? AND userid=? AND deletedat IS NULL"), now, now, id, topic, decodeUID(uid)) if err != nil { return err } if affected, _ := res.RowsAffected(); affected == 0 { return t.ErrNotFound } - if _, err = tx.ExecContext(ctx, `UPDATE im_group_members SET member_status=?,left_at=?,version=version+1,updated_at=? - WHERE topic=? AND user_id=? AND member_status=?`, t.GroupMemberStatusLeft, now, now, topic, decodeUID(uid), t.GroupMemberStatusActive); err != nil { + if _, err = tx.ExecContext(ctx, tenantShardSQL(id, `UPDATE im_group_members SET member_status=?,left_at=?,version=version+1,updated_at=? + WHERE tenant_id=? AND topic=? AND user_id=? AND member_status=?`), t.GroupMemberStatusLeft, now, now, id, topic, decodeUID(uid), t.GroupMemberStatusActive); err != nil { return err } - if err = recountGroupMembersAndBumpVersionTx(tx, topic); err != nil { + if err = recountGroupMembersAndBumpVersionTx(tx, id, topic); err != nil { return err } - if err = updateGroupTopicSubCountTx(tx, topic); err != nil { + if err = updateGroupTopicSubCountTx(tx, id, topic); err != nil { return err } return tx.Commit() @@ -1385,16 +1389,16 @@ func (a *adapter) TenantGroupRoleUpdate(id t.TenantID, topic string, actor, targ return err } defer tx.Rollback() - res, err := tx.ExecContext(ctx, "UPDATE im_subscriptions SET modegiven=?,modewant=?,updatedat=? WHERE topic=? AND userid=? AND deletedat IS NULL", - modeGiven.String(), modeGiven.String(), now, topic, decodeUID(target)) + res, err := tx.ExecContext(ctx, tenantShardSQL(id, "UPDATE im_subscriptions SET modegiven=?,modewant=?,updatedat=? WHERE tenant_id=? AND topic=? AND userid=? AND deletedat IS NULL"), + modeGiven.String(), modeGiven.String(), now, id, topic, decodeUID(target)) if err != nil { return err } if affected, _ := res.RowsAffected(); affected == 0 { return t.ErrNotFound } - res, err = tx.ExecContext(ctx, "UPDATE im_group_members SET role=?,version=version+1,updated_at=? WHERE topic=? AND user_id=? AND member_status=?", - role, now, topic, decodeUID(target), t.GroupMemberStatusActive) + res, err = tx.ExecContext(ctx, tenantShardSQL(id, "UPDATE im_group_members SET role=?,version=version+1,updated_at=? WHERE tenant_id=? AND topic=? AND user_id=? AND member_status=?"), + role, now, id, topic, decodeUID(target), t.GroupMemberStatusActive) if err != nil { return err } @@ -1430,16 +1434,16 @@ func (a *adapter) TenantGroupOwnerTransfer(id t.TenantID, topic string, previous role string }{{previousOwner, groupAdminAccessMode(), t.GroupMemberRoleAdmin}, {newOwner, t.ModeCFull, t.GroupMemberRoleOwner}} for _, update := range updates { - res, execErr := tx.ExecContext(ctx, "UPDATE im_subscriptions SET modegiven=?,modewant=?,updatedat=? WHERE topic=? AND userid=? AND deletedat IS NULL", - update.mode.String(), update.mode.String(), now, topic, decodeUID(update.uid)) + res, execErr := tx.ExecContext(ctx, tenantShardSQL(id, "UPDATE im_subscriptions SET modegiven=?,modewant=?,updatedat=? WHERE tenant_id=? AND topic=? AND userid=? AND deletedat IS NULL"), + update.mode.String(), update.mode.String(), now, id, topic, decodeUID(update.uid)) if execErr != nil { return execErr } if affected, _ := res.RowsAffected(); affected == 0 { return t.ErrNotFound } - res, execErr = tx.ExecContext(ctx, "UPDATE im_group_members SET role=?,version=version+1,updated_at=? WHERE topic=? AND user_id=? AND member_status=?", - update.role, now, topic, decodeUID(update.uid), t.GroupMemberStatusActive) + res, execErr = tx.ExecContext(ctx, tenantShardSQL(id, "UPDATE im_group_members SET role=?,version=version+1,updated_at=? WHERE tenant_id=? AND topic=? AND user_id=? AND member_status=?"), + update.role, now, id, topic, decodeUID(update.uid), t.GroupMemberStatusActive) if execErr != nil { return execErr } @@ -1447,7 +1451,7 @@ func (a *adapter) TenantGroupOwnerTransfer(id t.TenantID, topic string, previous return t.ErrNotFound } } - if _, err = tx.ExecContext(ctx, "UPDATE im_topics SET owner=?,updatedat=? WHERE name=?", decodeUID(newOwner), now, topic); err != nil { + if _, err = tx.ExecContext(ctx, tenantShardSQL(id, "UPDATE im_topics SET owner=?,updatedat=? WHERE tenant_id=? AND name=?"), decodeUID(newOwner), now, id, topic); err != nil { return err } args := []any{decodeUID(newOwner), now, topic} @@ -1504,48 +1508,48 @@ func insertGroupSettingsTx(tx *sqlx.Tx, settings *t.GroupSettings) error { return err } -func upsertGroupMemberTx(tx *sqlx.Tx, member *t.GroupMember) error { - _, err := tx.Exec(`INSERT INTO im_group_members - (topic,user_id,role,member_status,nickname,pinned,mute_notifications,join_source,joined_by,joined_at,version,created_at,updated_at) - VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?) +func upsertGroupMemberTx(tx *sqlx.Tx, tenantID t.TenantID, member *t.GroupMember) error { + _, err := tx.Exec(tenantShardSQL(tenantID, `INSERT INTO im_group_members + (tenant_id,topic,user_id,role,member_status,nickname,pinned,mute_notifications,join_source,joined_by,joined_at,version,created_at,updated_at) + VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?) ON DUPLICATE KEY UPDATE role=VALUES(role),member_status=VALUES(member_status),nickname=VALUES(nickname), pinned=VALUES(pinned),mute_notifications=VALUES(mute_notifications),join_source=VALUES(join_source), - joined_by=VALUES(joined_by),joined_at=VALUES(joined_at),left_at=NULL,removed_by=NULL,version=version+1,updated_at=VALUES(updated_at)`, - member.Topic, decodeUID(member.UserID), member.Role, member.MemberStatus, member.Nickname, member.Pinned, + joined_by=VALUES(joined_by),joined_at=VALUES(joined_at),left_at=NULL,removed_by=NULL,version=version+1,updated_at=VALUES(updated_at)`), + tenantID, member.Topic, decodeUID(member.UserID), member.Role, member.MemberStatus, member.Nickname, member.Pinned, member.MuteNotifications, member.JoinSource, nullableUID(member.JoinedBy), member.JoinedAt, member.Version, member.CreatedAt, member.UpdatedAt) return err } -func recountGroupMembersTx(tx *sqlx.Tx, topic string) error { - _, err := tx.Exec(`UPDATE im_group_settings SET member_count=(SELECT COUNT(*) FROM im_group_members - WHERE topic=? AND member_status=?),updated_at=CURRENT_TIMESTAMP(3) WHERE topic=?`, topic, t.GroupMemberStatusActive, topic) +func recountGroupMembersTx(tx *sqlx.Tx, tenantID t.TenantID, topic string) error { + _, err := tx.Exec(tenantShardSQL(tenantID, `UPDATE im_group_settings SET member_count=(SELECT COUNT(*) FROM im_group_members + WHERE tenant_id=? AND topic=? AND member_status=?),updated_at=CURRENT_TIMESTAMP(3) WHERE tenant_id=? AND topic=?`), tenantID, topic, t.GroupMemberStatusActive, tenantID, topic) return err } -func recountGroupMembersAndBumpVersionTx(tx *sqlx.Tx, topic string) error { - _, err := tx.Exec(`UPDATE im_group_settings SET member_count=(SELECT COUNT(*) FROM im_group_members - WHERE topic=? AND member_status=?),version=version+1,updated_at=CURRENT_TIMESTAMP(3) WHERE topic=?`, topic, t.GroupMemberStatusActive, topic) +func recountGroupMembersAndBumpVersionTx(tx *sqlx.Tx, tenantID t.TenantID, topic string) error { + _, err := tx.Exec(tenantShardSQL(tenantID, `UPDATE im_group_settings SET member_count=(SELECT COUNT(*) FROM im_group_members + WHERE tenant_id=? AND topic=? AND member_status=?),version=version+1,updated_at=CURRENT_TIMESTAMP(3) WHERE tenant_id=? AND topic=?`), tenantID, topic, t.GroupMemberStatusActive, tenantID, topic) return err } -func updateGroupTopicSubCountTx(tx *sqlx.Tx, topic string) error { - _, err := tx.Exec(`UPDATE im_topics SET subcnt=(SELECT COUNT(*) FROM im_subscriptions - WHERE topic IN (?,?) AND deletedat IS NULL) WHERE name=?`, topic, t.GrpToChn(topic), topic) +func updateGroupTopicSubCountTx(tx *sqlx.Tx, tenantID t.TenantID, topic string) error { + _, err := tx.Exec(tenantShardSQL(tenantID, `UPDATE im_topics SET subcnt=(SELECT COUNT(*) FROM im_subscriptions + WHERE tenant_id=? AND topic IN (?,?) AND deletedat IS NULL) WHERE tenant_id=? AND name=?`), tenantID, topic, t.GrpToChn(topic), tenantID, topic) return err } -func syncGroupMuteAllTx(tx *sqlx.Tx, topic string, muteAll bool, now time.Time) error { +func syncGroupMuteAllTx(tx *sqlx.Tx, tenantID t.TenantID, topic string, muteAll bool, now time.Time) error { mode := t.ModeCPublic if muteAll { mode = t.ModeCPublic &^ t.ModeWrite } - _, err := tx.Exec(`UPDATE im_subscriptions AS s + _, err := tx.Exec(tenantShardSQL(tenantID, `UPDATE im_subscriptions AS s JOIN im_group_members AS gm ON gm.topic=s.topic AND gm.user_id=s.userid SET s.modewant=?,s.modegiven=?,s.updatedat=?, gm.version=gm.version+1,gm.updated_at=? WHERE s.topic=? AND s.deletedat IS NULL - AND gm.member_status=? AND gm.role=?`, + AND gm.member_status=? AND gm.role=?`), mode.String(), mode.String(), now, now, topic, t.GroupMemberStatusActive, t.GroupMemberRoleMember) return err } @@ -1656,8 +1660,8 @@ func (a *adapter) groupFileByQuery(ctx context.Context, query string, args ...an return nil, t.ErrNotFound } -func (a *adapter) recountGroupMembers(ctx context.Context, topic string) error { - _, err := a.db.ExecContext(ctx, "UPDATE im_group_settings SET member_count=(SELECT COUNT(*) FROM im_group_members WHERE topic=? AND member_status=?),updated_at=CURRENT_TIMESTAMP(3) WHERE topic=?", topic, t.GroupMemberStatusActive, topic) +func (a *adapter) recountGroupMembers(ctx context.Context, tenantID t.TenantID, topic string) error { + _, err := a.db.ExecContext(ctx, tenantShardSQL(tenantID, "UPDATE im_group_settings SET member_count=(SELECT COUNT(*) FROM im_group_members WHERE tenant_id=? AND topic=? AND member_status=?),updated_at=CURRENT_TIMESTAMP(3) WHERE tenant_id=? AND topic=?"), tenantID, topic, t.GroupMemberStatusActive, tenantID, topic) return err } diff --git a/server/db/mysql/tenant_infrastructure.go b/server/db/mysql/tenant_infrastructure.go index 320a81b..5a6201e 100644 --- a/server/db/mysql/tenant_infrastructure.go +++ b/server/db/mysql/tenant_infrastructure.go @@ -4,15 +4,12 @@ package mysql import ( - "context" - "database/sql" "encoding/json" "strconv" "strings" "time" "github.com/jmoiron/sqlx" - "github.com/tinode/chat/server/store" t "github.com/tinode/chat/server/store/types" ) @@ -26,214 +23,6 @@ func (a *adapter) infrastructureAdapter(id t.TenantID) (*adapter, error) { return a, nil } -func (a *adapter) TenantHTTPIdempotencyBegin(id t.TenantID, record *t.HTTPIdempotencyRecord) (*t.HTTPIdempotencyRecord, bool, error) { - tenantAdp, err := a.infrastructureAdapter(id) - if err != nil { - return nil, false, err - } - if record == nil || record.UserID.IsZero() || record.Method == "" || record.Route == "" || record.Key == "" || record.BodyHash == "" { - return nil, false, t.ErrMalformed - } - if tenantAdp != a { - return tenantAdp.TenantHTTPIdempotencyBegin(id, record) - } - now := t.TimeNow() - if record.ExpiresAt.IsZero() { - record.ExpiresAt = now.Add(24 * time.Hour) - } - _, err = a.db.Exec(`INSERT INTO im_http_idempotency - (tenant_id,user_id,method,route,idempotency_key,body_hash,state,response_status,response_body,expires_at,created_at,updated_at) - VALUES(?,?,?,?,?,?,?,?,?,?,?,?)`, id, decodeUID(record.UserID), record.Method, record.Route, record.Key, - record.BodyHash, t.HTTPIdempotencyProcessing, 0, nil, record.ExpiresAt, now, now) - if err == nil { - record.State = t.HTTPIdempotencyProcessing - record.CreatedAt = now - record.UpdatedAt = now - return record, true, nil - } - if !isDupe(err) { - return nil, false, err - } - existing, err := a.httpIdempotencyGet(id, record.UserID, record.Method, record.Route, record.Key) - if err == nil && !existing.ExpiresAt.After(now) { - res, deleteErr := a.db.Exec(`DELETE FROM im_http_idempotency - WHERE tenant_id=? AND user_id=? AND method=? AND route=? AND idempotency_key=? AND expires_at<=?`, - id, decodeUID(record.UserID), record.Method, record.Route, record.Key, now) - if deleteErr != nil { - return nil, false, deleteErr - } - if deleted, _ := res.RowsAffected(); deleted > 0 { - return a.TenantHTTPIdempotencyBegin(id, record) - } - current, getErr := a.httpIdempotencyGet(id, record.UserID, record.Method, record.Route, record.Key) - return current, false, getErr - } - return existing, false, err -} - -func (a *adapter) TenantHTTPIdempotencyComplete(id t.TenantID, record *t.HTTPIdempotencyRecord) error { - tenantAdp, err := a.infrastructureAdapter(id) - if err != nil { - return err - } - if record == nil || record.UserID.IsZero() { - return t.ErrMalformed - } - if tenantAdp != a { - return tenantAdp.TenantHTTPIdempotencyComplete(id, record) - } - now := t.TimeNow() - res, err := a.db.Exec(`UPDATE im_http_idempotency SET state=?,response_status=?,response_body=?,updated_at=? - WHERE tenant_id=? AND user_id=? AND method=? AND route=? AND idempotency_key=? AND body_hash=? AND state=?`, - t.HTTPIdempotencyCompleted, record.ResponseStatus, record.ResponseBody, now, id, decodeUID(record.UserID), - record.Method, record.Route, record.Key, record.BodyHash, t.HTTPIdempotencyProcessing) - if err != nil { - return err - } - if count, _ := res.RowsAffected(); count == 0 { - return t.ErrNotFound - } - return nil -} - -func (a *adapter) TenantHTTPIdempotencyAbort(id t.TenantID, record *t.HTTPIdempotencyRecord) error { - tenantAdp, err := a.infrastructureAdapter(id) - if err != nil { - return err - } - if record == nil || record.UserID.IsZero() { - return t.ErrMalformed - } - if tenantAdp != a { - return tenantAdp.TenantHTTPIdempotencyAbort(id, record) - } - _, err = a.db.Exec(`DELETE FROM im_http_idempotency - WHERE tenant_id=? AND user_id=? AND method=? AND route=? AND idempotency_key=? AND body_hash=? AND state=?`, - id, decodeUID(record.UserID), record.Method, record.Route, record.Key, record.BodyHash, t.HTTPIdempotencyProcessing) - return err -} - -func (a *adapter) httpIdempotencyGet(tenantID t.TenantID, uid t.Uid, method, route, key string) (*t.HTTPIdempotencyRecord, error) { - row := struct { - UserID int64 `db:"user_id"` - Method string - Route string - Key string `db:"idempotency_key"` - BodyHash string `db:"body_hash"` - State string - ResponseStatus int `db:"response_status"` - ResponseBody []byte `db:"response_body"` - CreatedAt time.Time `db:"created_at"` - UpdatedAt time.Time `db:"updated_at"` - ExpiresAt time.Time `db:"expires_at"` - }{} - err := a.db.Get(&row, `SELECT user_id,method,route,idempotency_key,body_hash,state,response_status,response_body,created_at,updated_at,expires_at - FROM im_http_idempotency WHERE tenant_id=? AND user_id=? AND method=? AND route=? AND idempotency_key=?`, - tenantID, decodeUID(uid), method, route, key) - if err == sql.ErrNoRows { - return nil, t.ErrNotFound - } - if err != nil { - return nil, err - } - return &t.HTTPIdempotencyRecord{UserID: store.EncodeUid(row.UserID), Method: row.Method, Route: row.Route, - Key: row.Key, BodyHash: row.BodyHash, State: row.State, ResponseStatus: row.ResponseStatus, - ResponseBody: row.ResponseBody, CreatedAt: row.CreatedAt, UpdatedAt: row.UpdatedAt, ExpiresAt: row.ExpiresAt}, nil -} - -func (a *adapter) TenantKafkaConsumeBegin(id t.TenantID, record *t.KafkaConsumeRecord, deadline time.Time) (*t.KafkaConsumeRecord, bool, error) { - tenantAdp, err := a.infrastructureAdapter(id) - if err != nil { - return nil, false, err - } - if record == nil || record.EventID == "" || record.ConsumerGroup == "" || record.SourceTopic == "" { - return nil, false, t.ErrMalformed - } - if tenantAdp != a { - return tenantAdp.TenantKafkaConsumeBegin(id, record, deadline) - } - now := t.TimeNow() - _, err = a.db.Exec(`INSERT INTO im_kafka_consume_log - (event_id,consumer_group,source_topic,event_type,biz_type,biz_id,status,processing_deadline,created_at,updated_at) - VALUES(?,?,?,?,?,?,?,?,?,?)`, record.EventID, record.ConsumerGroup, record.SourceTopic, record.EventType, - record.BizType, record.BizID, t.KafkaConsumeProcessing, deadline, now, now) - if err == nil { - record.Status = t.KafkaConsumeProcessing - record.ProcessingDeadline = &deadline - record.CreatedAt = now - record.UpdatedAt = now - return record, true, nil - } - if !isDupe(err) { - return nil, false, err - } - existing, err := a.kafkaConsumeGet(record.EventID, record.ConsumerGroup) - if err != nil { - return nil, false, err - } - if existing.Status == t.KafkaConsumeSucceeded || existing.Status == t.KafkaConsumeDead { - return existing, false, nil - } - if existing.Status == t.KafkaConsumeProcessing && existing.ProcessingDeadline != nil && existing.ProcessingDeadline.After(now) { - return existing, false, nil - } - res, err := a.db.Exec(`UPDATE im_kafka_consume_log SET status=?,source_topic=?,event_type=?,biz_type=?,biz_id=?, - processing_deadline=?,updated_at=? WHERE event_id=? AND consumer_group=? AND status<>?`, - t.KafkaConsumeProcessing, record.SourceTopic, record.EventType, record.BizType, record.BizID, - deadline, now, record.EventID, record.ConsumerGroup, t.KafkaConsumeSucceeded) - if err != nil { - return nil, false, err - } - if affected, _ := res.RowsAffected(); affected == 0 { - current, getErr := a.kafkaConsumeGet(record.EventID, record.ConsumerGroup) - return current, false, getErr - } - current, err := a.kafkaConsumeGet(record.EventID, record.ConsumerGroup) - return current, err == nil, err -} - -func (a *adapter) TenantKafkaConsumeSucceed(id t.TenantID, eventID, consumerGroup string, now time.Time) error { - tenantAdp, err := a.infrastructureAdapter(id) - if err != nil { - return err - } - if eventID == "" || consumerGroup == "" { - return t.ErrMalformed - } - if tenantAdp != a { - return tenantAdp.TenantKafkaConsumeSucceed(id, eventID, consumerGroup, now) - } - res, err := a.db.Exec(`UPDATE im_kafka_consume_log SET status=?,succeeded_at=?,processing_deadline=NULL,updated_at=? - WHERE event_id=? AND consumer_group=?`, t.KafkaConsumeSucceeded, now, now, eventID, consumerGroup) - if err != nil { - return err - } - if affected, _ := res.RowsAffected(); affected == 0 { - return t.ErrNotFound - } - return nil -} - -func (a *adapter) TenantKafkaConsumeFail(id t.TenantID, eventID, consumerGroup, reason, message string, now time.Time) (*t.KafkaConsumeRecord, error) { - tenantAdp, err := a.infrastructureAdapter(id) - if err != nil { - return nil, err - } - if eventID == "" || consumerGroup == "" { - return nil, t.ErrMalformed - } - if tenantAdp != a { - return tenantAdp.TenantKafkaConsumeFail(id, eventID, consumerGroup, reason, message, now) - } - _, err = a.db.Exec(`UPDATE im_kafka_consume_log SET status=?,retry_count=retry_count+1,last_reason=?, - last_error=?,processing_deadline=NULL,updated_at=? WHERE event_id=? AND consumer_group=?`, - t.KafkaConsumeFailed, truncateKafkaString(reason, 64), truncateKafkaString(message, 1000), now, eventID, consumerGroup) - if err != nil { - return nil, err - } - return a.kafkaConsumeGet(eventID, consumerGroup) -} - func (a *adapter) TenantKafkaConsumeDead(id t.TenantID, dead *t.KafkaDeadLetter, now time.Time) error { tenantAdp, err := a.infrastructureAdapter(id) if err != nil { @@ -245,12 +34,7 @@ func (a *adapter) TenantKafkaConsumeDead(id t.TenantID, dead *t.KafkaDeadLetter, if tenantAdp != a { return tenantAdp.TenantKafkaConsumeDead(id, dead, now) } - tx, err := a.db.BeginTxx(context.Background(), nil) - if err != nil { - return err - } - defer tx.Rollback() - if _, err = tx.Exec(`INSERT INTO im_kafka_dead_letters + if _, err = a.db.Exec(`INSERT INTO im_kafka_dead_letters (dead_id,event_id,parent_event_id,source_topic,consumer_group,event_type,biz_type,biz_id,envelope, retry_count,last_reason,last_error,status,replay_count,created_at,updated_at) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) @@ -260,13 +44,7 @@ func (a *adapter) TenantKafkaConsumeDead(id t.TenantID, dead *t.KafkaDeadLetter, truncateKafkaString(dead.LastError, 1000), t.KafkaDeadStatusDead, dead.ReplayCount, now, now); err != nil { return err } - if _, err = tx.Exec(`UPDATE im_kafka_consume_log SET status=?,dead_at=?,processing_deadline=NULL,last_reason=?, - last_error=?,updated_at=? WHERE event_id=? AND consumer_group=?`, - t.KafkaConsumeDead, now, truncateKafkaString(dead.LastReason, 64), truncateKafkaString(dead.LastError, 1000), - now, dead.EventID, dead.ConsumerGroup); err != nil { - return err - } - return tx.Commit() + return nil } func (a *adapter) TenantKafkaDeadLetterList(id t.TenantID, opts t.KafkaDeadLetterListOptions) (*t.KafkaDeadLetterPage, error) { @@ -390,22 +168,7 @@ func (a *adapter) TenantKafkaDeadLetterMarkReplay(id t.TenantID, deadID, resolve if tenantAdp != a { return tenantAdp.TenantKafkaDeadLetterMarkReplay(id, deadID, resolvedBy, note, now) } - tx, err := a.db.BeginTxx(context.Background(), nil) - if err != nil { - return nil, err - } - defer tx.Rollback() - var row struct { - EventID string `db:"event_id"` - ConsumerGroup string `db:"consumer_group"` - } - if err = tx.Get(&row, `SELECT event_id,consumer_group FROM im_kafka_dead_letters WHERE dead_id=? FOR UPDATE`, deadID); err != nil { - if err == sql.ErrNoRows { - return nil, t.ErrNotFound - } - return nil, err - } - res, err := tx.Exec(`UPDATE im_kafka_dead_letters SET status=?,replay_count=replay_count+1,resolved_by=?, + res, err := a.db.Exec(`UPDATE im_kafka_dead_letters SET status=?,replay_count=replay_count+1,resolved_by=?, resolved_at=?,resolve_note=?,updated_at=? WHERE dead_id=? AND status IN (?,?)`, t.KafkaDeadStatusPendingReplay, truncateKafkaString(resolvedBy, 64), now, truncateKafkaString(note, 1000), now, deadID, t.KafkaDeadStatusDead, t.KafkaDeadStatusPendingReplay) @@ -415,13 +178,6 @@ func (a *adapter) TenantKafkaDeadLetterMarkReplay(id t.TenantID, deadID, resolve if affected, _ := res.RowsAffected(); affected == 0 { return nil, t.ErrNotFound } - if _, err = tx.Exec(`DELETE FROM im_kafka_consume_log WHERE event_id=? AND consumer_group=?`, - row.EventID, row.ConsumerGroup); err != nil { - return nil, err - } - if err = tx.Commit(); err != nil { - return nil, err - } return a.TenantKafkaDeadLetterGet(id, deadID) } @@ -469,39 +225,6 @@ func scanKafkaDeadLetter(rows *sqlx.Rows) (*t.KafkaDeadLetter, error) { }, nil } -func (a *adapter) kafkaConsumeGet(eventID, consumerGroup string) (*t.KafkaConsumeRecord, error) { - row := struct { - EventID string `db:"event_id"` - ConsumerGroup string `db:"consumer_group"` - SourceTopic string `db:"source_topic"` - EventType string `db:"event_type"` - BizType string `db:"biz_type"` - BizID string `db:"biz_id"` - Status string `db:"status"` - RetryCount int `db:"retry_count"` - LastReason string `db:"last_reason"` - LastError string `db:"last_error"` - ProcessingDeadline *time.Time `db:"processing_deadline"` - SucceededAt *time.Time `db:"succeeded_at"` - DeadAt *time.Time `db:"dead_at"` - CreatedAt time.Time `db:"created_at"` - UpdatedAt time.Time `db:"updated_at"` - }{} - err := a.db.Get(&row, `SELECT event_id,consumer_group,source_topic,event_type,biz_type,biz_id,status,retry_count, - last_reason,last_error,processing_deadline,succeeded_at,dead_at,created_at,updated_at - FROM im_kafka_consume_log WHERE event_id=? AND consumer_group=?`, eventID, consumerGroup) - if err == sql.ErrNoRows { - return nil, t.ErrNotFound - } - if err != nil { - return nil, err - } - return &t.KafkaConsumeRecord{EventID: row.EventID, ConsumerGroup: row.ConsumerGroup, SourceTopic: row.SourceTopic, - EventType: row.EventType, BizType: row.BizType, BizID: row.BizID, Status: row.Status, RetryCount: row.RetryCount, - LastReason: row.LastReason, LastError: row.LastError, ProcessingDeadline: row.ProcessingDeadline, - SucceededAt: row.SucceededAt, DeadAt: row.DeadAt, CreatedAt: row.CreatedAt, UpdatedAt: row.UpdatedAt}, nil -} - func truncateKafkaString(value string, limit int) string { if limit <= 0 { return "" @@ -519,10 +242,10 @@ func insertSystemNotificationTx(tx *sqlx.Tx, tenantID t.TenantID, uid t.Uid, eve return err } now := t.TimeNow() - _, err = tx.Exec(`INSERT INTO im_system_notifications + _, err = tx.Exec(tenantShardSQL(tenantID, `INSERT INTO im_system_notifications (tenant_id,user_id,event_id,event_type,title,summary,action,created_at,updated_at) VALUES(?,?,?,?,?,?,?,?,?) ON DUPLICATE KEY UPDATE event_type=VALUES(event_type),title=VALUES(title), - summary=VALUES(summary),action=VALUES(action),updated_at=VALUES(updated_at)`, tenantID, decodeUID(uid), eventID, + summary=VALUES(summary),action=VALUES(action),updated_at=VALUES(updated_at)`), tenantID, decodeUID(uid), eventID, eventType, title, summary, encoded, now, now) return err } diff --git a/server/db/mysql/tenant_security_audit.go b/server/db/mysql/tenant_security_audit.go index ae46f9a..4e2fe20 100644 --- a/server/db/mysql/tenant_security_audit.go +++ b/server/db/mysql/tenant_security_audit.go @@ -68,21 +68,7 @@ func (a *adapter) TenantLoginFailureCreate(id t.TenantID, event *t.LoginFailureE if tenantAdp != a { return tenantAdp.TenantLoginFailureCreate(id, event) } - metadata, err := json.Marshal(event.Metadata) - if err != nil { - return err - } - var userID any - if !event.UserID.IsZero() { - userID = decodeUID(event.UserID) - } - _, err = a.db.Exec(`INSERT INTO im_login_failure_logs - (user_id,account_type,account_hash,account_mask,failure_type,reason,detail,ip,region,device_id,platform,app_version,user_agent,risk_level,risk_action,request_id,attempt_at,metadata) - VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)`, - userID, event.AccountType, event.AccountHash, event.AccountMask, event.FailureType, event.Reason, event.Detail, - event.IP, event.Region, event.DeviceID, event.Platform, event.AppVersion, event.UserAgent, - event.RiskLevel, event.RiskAction, event.RequestID, event.AttemptAt, nullableJSON(metadata)) - return err + return nil } func (a *adapter) TenantLoginAuditClose(id t.TenantID, sessionID string, at time.Time) error { diff --git a/server/db/mysql/tenant_tables.go b/server/db/mysql/tenant_tables.go index 01282d5..4f1f0af 100644 --- a/server/db/mysql/tenant_tables.go +++ b/server/db/mysql/tenant_tables.go @@ -5,10 +5,18 @@ package mysql import ( "fmt" + "strings" t "github.com/tinode/chat/server/store/types" ) +func tenantShardSQL(tenantID t.TenantID, query string) string { + query = strings.ReplaceAll(query, "im_topics", tenantTopicsTable(tenantID)) + query = strings.ReplaceAll(query, "im_group_members", tenantGroupMembersTable(tenantID)) + query = strings.ReplaceAll(query, "im_subscriptions", tenantSubscriptionsTable(tenantID)) + return strings.ReplaceAll(query, "im_system_notifications", tenantSystemNotificationsTable(tenantID)) +} + func tenantTablePrefix(tenantID t.TenantID) string { return fmt.Sprintf("im_%03d_", tenantID) } @@ -21,6 +29,22 @@ func tenantMessagesTable(tenantID t.TenantID) string { return tenantTablePrefix(tenantID) + "messages" } +func tenantTopicsTable(tenantID t.TenantID) string { + return tenantTablePrefix(tenantID) + "topics" +} + +func tenantGroupMembersTable(tenantID t.TenantID) string { + return tenantTablePrefix(tenantID) + "group_members" +} + +func tenantSubscriptionsTable(tenantID t.TenantID) string { + return tenantTablePrefix(tenantID) + "subscriptions" +} + +func tenantSystemNotificationsTable(tenantID t.TenantID) string { + return tenantTablePrefix(tenantID) + "system_notifications" +} + func tenantGroupMessagesTable(tenantID t.TenantID, month string) (string, bool) { if !validShardMonth(month) { return "", false diff --git a/server/db/mysql/tests/mysql_test.go b/server/db/mysql/tests/mysql_test.go index ae0b98f..96e3e9f 100644 --- a/server/db/mysql/tests/mysql_test.go +++ b/server/db/mysql/tests/mysql_test.go @@ -61,7 +61,7 @@ func TestGroupBusinessTablesCreated(t *testing.T) { tables := []string{ "im_group_settings", "im_group_members", "im_group_invitations", "im_group_join_requests", "im_group_announcements", "im_group_announcement_reads", "im_group_mutes", "im_group_qrcodes", - "im_group_files", "im_group_admin_logs", "im_group_idempotency", + "im_group_files", "im_group_admin_logs", } args := make([]any, len(tables)) for i, table := range tables { @@ -103,8 +103,8 @@ func TestHTTPInfrastructureTablesCreated(t *testing.T) { if err := db.Get(&count, query, args...); err != nil { t.Fatal(err) } - if count != len(tables) { - t.Fatalf("HTTP infrastructure table count: got %d, want %d", count, len(tables)) + if count != 0 { + t.Fatalf("HTTP idempotency must not create MySQL table: got %d", count) } query = "SELECT COUNT(*) FROM information_schema.columns " + @@ -112,8 +112,8 @@ func TestHTTPInfrastructureTablesCreated(t *testing.T) { if err := db.Get(&count, query, args...); err != nil { t.Fatal(err) } - if count != len(tables) { - t.Fatalf("HTTP infrastructure tenant_id columns: got %d, want %d", count, len(tables)) + if count != 0 { + t.Fatalf("HTTP idempotency tenant_id columns should not exist: got %d", count) } } diff --git a/server/im_http.go b/server/im_http.go index 034cf2d..9db0ad5 100644 --- a/server/im_http.go +++ b/server/im_http.go @@ -52,6 +52,7 @@ const ( imReasonSchemaUnsupported = "SCHEMA_UNSUPPORTED" imReasonIdempotencyConflict = "IDEMPOTENCY_CONFLICT" imReasonIdempotencyPending = "IDEMPOTENCY_IN_PROGRESS" + imReasonIdempotencyService = "IDEMPOTENCY_UNAVAILABLE" imReasonInternal = "INTERNAL_ERROR" imReasonUnsupportedMedia = "UNSUPPORTED_MEDIA_TYPE" imReasonFileUnavailable = "FILE_UNAVAILABLE" @@ -69,6 +70,75 @@ const ( registrationTypePersonal = 1 ) +const ( + imHTTPIdempotencyTTL = 24 * time.Hour + imHTTPIdempotencyNamespace = "http_idempotency" +) + +type imHTTPIdempotencyReplay struct { + BodyHash string `json:"body_hash"` + ResponseStatus int `json:"response_status"` + ResponseBody []byte `json:"response_body"` +} + +var errIMHTTPIdempotencyConflict = errors.New("http idempotency conflict") + +var ( + beginIMHTTPIdempotency = redisBeginIMHTTPIdempotency + completeIMHTTPIdempotency = redisCompleteIMHTTPIdempotency + abortIMHTTPIdempotency = redisAbortIMHTTPIdempotency +) + +func imHTTPIdempotencyID(tenantID types.TenantID, uid types.Uid, method, route, key string) string { + return strconv.FormatInt(int64(tenantID), 10) + ":" + uid.String() + ":" + method + ":" + route + ":" + key +} + +func redisBeginIMHTTPIdempotency(tenantID types.TenantID, uid types.Uid, method, route, key, bodyHash string) (*imHTTPIdempotencyReplay, bool, bool, error) { + id := imHTTPIdempotencyID(tenantID, uid, method, route, key) + claimed, sameValue, enabled, err := store.ClaimShortStateValue(imHTTPIdempotencyNamespace, id, bodyHash, imHTTPIdempotencyTTL) + if err != nil || !enabled || claimed { + return nil, claimed, enabled, err + } + if !sameValue { + return nil, false, true, errIMHTTPIdempotencyConflict + } + raw, found, enabled, err := store.GetShortStateValue(imHTTPIdempotencyNamespace, id+":response") + if err != nil || !enabled || !found { + return nil, false, enabled, err + } + var replay imHTTPIdempotencyReplay + if err = json.Unmarshal([]byte(raw), &replay); err != nil { + _, _ = store.DeleteShortState(imHTTPIdempotencyNamespace, id+":response") + return nil, false, true, err + } + if replay.BodyHash != bodyHash { + return nil, false, true, nil + } + return &replay, false, true, nil +} + +func redisCompleteIMHTTPIdempotency(tenantID types.TenantID, uid types.Uid, method, route, key, bodyHash string, status int, body []byte) error { + id := imHTTPIdempotencyID(tenantID, uid, method, route, key) + payload, err := json.Marshal(imHTTPIdempotencyReplay{BodyHash: bodyHash, ResponseStatus: status, ResponseBody: body}) + if err != nil { + return err + } + enabled, err := store.SetShortStateValue(imHTTPIdempotencyNamespace, id+":response", string(payload), imHTTPIdempotencyTTL) + if !enabled { + return errors.New("http idempotency redis unavailable") + } + return err +} + +func redisAbortIMHTTPIdempotency(tenantID types.TenantID, uid types.Uid, method, route, key string) error { + id := imHTTPIdempotencyID(tenantID, uid, method, route, key) + if _, err := store.DeleteShortState(imHTTPIdempotencyNamespace, id); err != nil { + return err + } + _, err := store.DeleteShortState(imHTTPIdempotencyNamespace, id+":response") + return err +} + func reviewIMHTTPText(wrt http.ResponseWriter, req *http.Request, principal *imPrincipal, scene, topic, text string) bool { text = strings.TrimSpace(text) if text == "" { @@ -232,53 +302,54 @@ func serveIMHTTPIdempotent(wrt http.ResponseWriter, req *http.Request, next http req.Body = io.NopCloser(bytes.NewReader(body)) hashInput := append([]byte(req.URL.RawQuery+"\x00"), body...) bodyHash := sha256.Sum256(hashInput) - record := &types.HTTPIdempotencyRecord{ - UserID: principal.Uid, Method: req.Method, Route: req.URL.Path, Key: key, - BodyHash: fmt.Sprintf("%x", bodyHash[:]), ExpiresAt: time.Now().UTC().Add(24 * time.Hour), + bodyHashText := fmt.Sprintf("%x", bodyHash[:]) + existing, acquired, enabled, err := beginIMHTTPIdempotency(principal.TenantID, principal.Uid, req.Method, req.URL.Path, key, bodyHashText) + if errors.Is(err, errIMHTTPIdempotencyConflict) { + writeIMError(wrt, req, http.StatusConflict, 30007, imReasonIdempotencyConflict, "Idempotency-Key was used with a different request", nil) + return } - existing, acquired, err := store.HTTPInfrastructure.BeginIdempotency(principal.TenantID, record) if err != nil { - logs.Warn.Printf("im http idempotency begin failed request_id=%s tenant=%d user=%s method=%s route=%s err=%v", + logs.Warn.Printf("im http redis idempotency begin failed request_id=%s tenant=%d user=%s method=%s route=%s err=%v", req.Header.Get("X-Request-ID"), principal.TenantID, principal.Uid.UserId(), req.Method, req.URL.Path, err) - writeIMError(wrt, req, http.StatusInternalServerError, 90000, imReasonInternal, "internal error", nil) + writeIMError(wrt, req, http.StatusServiceUnavailable, 30050, imReasonIdempotencyService, "idempotency service unavailable", nil) + return + } + if !enabled { + logs.Warn.Printf("im http redis idempotency unavailable request_id=%s tenant=%d user=%s method=%s route=%s", + req.Header.Get("X-Request-ID"), principal.TenantID, principal.Uid.UserId(), req.Method, req.URL.Path) + writeIMError(wrt, req, http.StatusServiceUnavailable, 30050, imReasonIdempotencyService, "idempotency service unavailable", nil) return } if !acquired { - if existing == nil || existing.BodyHash != record.BodyHash { - writeIMError(wrt, req, http.StatusConflict, 30007, imReasonIdempotencyConflict, "Idempotency-Key was used with a different request", nil) + if existing == nil { + writeIMError(wrt, req, http.StatusConflict, 30008, imReasonIdempotencyPending, "request with this Idempotency-Key is still processing", nil) return } - if existing.State == types.HTTPIdempotencyCompleted { - wrt.Header().Set("Content-Type", "application/json; charset=utf-8") - wrt.Header().Set("X-Idempotent-Replay", "true") - wrt.WriteHeader(existing.ResponseStatus) - _, _ = wrt.Write(existing.ResponseBody) - return - } - writeIMError(wrt, req, http.StatusConflict, 30008, imReasonIdempotencyPending, "request with this Idempotency-Key is still processing", nil) + wrt.Header().Set("Content-Type", "application/json; charset=utf-8") + wrt.Header().Set("X-Idempotent-Replay", "true") + wrt.WriteHeader(existing.ResponseStatus) + _, _ = wrt.Write(existing.ResponseBody) return } buffered := newIMBufferedResponseWriter() next.ServeHTTP(buffered, req) - record.ResponseStatus = buffered.status - record.ResponseBody = append([]byte(nil), buffered.body.Bytes()...) if buffered.status >= http.StatusInternalServerError { - if abortErr := store.HTTPInfrastructure.AbortIdempotency(principal.TenantID, record); abortErr != nil { - logs.Warn.Printf("im http idempotency abort failed request_id=%s tenant=%d user=%s method=%s route=%s err=%v", + if abortErr := abortIMHTTPIdempotency(principal.TenantID, principal.Uid, req.Method, req.URL.Path, key); abortErr != nil { + logs.Warn.Printf("im http redis idempotency abort failed request_id=%s tenant=%d user=%s method=%s route=%s err=%v", req.Header.Get("X-Request-ID"), principal.TenantID, principal.Uid.UserId(), req.Method, req.URL.Path, abortErr) } buffered.flushTo(wrt) return } - if err = store.HTTPInfrastructure.CompleteIdempotency(principal.TenantID, record); err != nil { - logs.Warn.Printf("im http idempotency complete failed request_id=%s tenant=%d user=%s method=%s route=%s status=%d err=%v", + if err = completeIMHTTPIdempotency(principal.TenantID, principal.Uid, req.Method, req.URL.Path, key, bodyHashText, buffered.status, append([]byte(nil), buffered.body.Bytes()...)); err != nil { + logs.Warn.Printf("im http redis idempotency complete failed request_id=%s tenant=%d user=%s method=%s route=%s status=%d err=%v", req.Header.Get("X-Request-ID"), principal.TenantID, principal.Uid.UserId(), req.Method, req.URL.Path, buffered.status, err) - if abortErr := store.HTTPInfrastructure.AbortIdempotency(principal.TenantID, record); abortErr != nil { - logs.Warn.Printf("im http idempotency abort after complete failed request_id=%s tenant=%d user=%s method=%s route=%s err=%v", + if abortErr := abortIMHTTPIdempotency(principal.TenantID, principal.Uid, req.Method, req.URL.Path, key); abortErr != nil { + logs.Warn.Printf("im http redis idempotency abort after complete failed request_id=%s tenant=%d user=%s method=%s route=%s err=%v", req.Header.Get("X-Request-ID"), principal.TenantID, principal.Uid.UserId(), req.Method, req.URL.Path, abortErr) } - writeIMError(wrt, req, http.StatusInternalServerError, 90000, imReasonInternal, "internal error", nil) + writeIMError(wrt, req, http.StatusServiceUnavailable, 30050, imReasonIdempotencyService, "idempotency service unavailable", nil) return } buffered.flushTo(wrt) @@ -2412,7 +2483,8 @@ func serveIMSystemNotificationRead(wrt http.ResponseWriter, req *http.Request) { } readAt = parsed.UTC().Round(time.Millisecond) } - if err := store.Security.MarkSystemNotificationRead(principal.TenantID, principal.Uid, eventID, readAt); err != nil { + notification, err := store.Security.MarkSystemNotificationRead(principal.TenantID, principal.Uid, eventID, readAt) + if err != nil { if errors.Is(err, types.ErrNotFound) { writeIMError(wrt, req, http.StatusNotFound, 30044, imReasonChallengeExpired, "notification not found", nil) return @@ -2420,7 +2492,9 @@ func serveIMSystemNotificationRead(wrt http.ResponseWriter, req *http.Request) { writeIMError(wrt, req, http.StatusInternalServerError, 90000, imReasonInternal, "internal error", nil) return } - writeIMSuccess(wrt, req, map[string]any{"status": "ok", "id": eventID, "read_at": readAt.Format(time.RFC3339Nano)}) + payload := systemNotificationPayload(*notification) + payload["status"] = "ok" + writeIMSuccess(wrt, req, payload) } func serveIMFavorites(wrt http.ResponseWriter, req *http.Request) { @@ -7978,7 +8052,7 @@ func favoritePayload(item types.FavoriteItem) map[string]any { "favorite_id": item.FavoriteID, "type": item.ItemType, "snapshot": item.Snapshot, - "source": map[string]any{"topic": item.SourceTopic, "seq": item.SourceSeq}, + "source": favoriteSourcePayload(item.SourceTopic, item.SourceSeq, types.ZeroUid), "collected_at": item.CollectedAt.UTC().Format(time.RFC3339Nano), "availability": item.Availability, } @@ -8015,7 +8089,27 @@ func favoritePayloadForPrincipal(principal *imPrincipal, item types.FavoriteItem item.Availability = "removed" } } - return favoritePayload(item), true + return favoritePayloadForViewer(principal.Uid, item), true +} + +func favoritePayloadForViewer(viewer types.Uid, item types.FavoriteItem) map[string]any { + payload := favoritePayload(item) + payload["source"] = favoriteSourcePayload(item.SourceTopic, item.SourceSeq, viewer) + return payload +} + +func favoriteSourcePayload(topic string, seq int64, viewer types.Uid) map[string]any { + source := map[string]any{"topic": topic, "seq": seq} + if viewer.IsZero() || types.GetTopicCat(topic) != types.TopicCatP2P { + return source + } + peerTopic, err := types.P2PNameForUser(viewer, topic) + if err != nil || strings.TrimSpace(peerTopic) == "" { + return source + } + source["peer_user_id"] = peerTopic + source["display_topic"] = peerTopic + return source } func legacyFavoriteSnapshotOnly(item types.FavoriteItem) bool { diff --git a/server/im_http_test.go b/server/im_http_test.go index f5cde7b..65f209e 100644 --- a/server/im_http_test.go +++ b/server/im_http_test.go @@ -41,7 +41,6 @@ type testVerificationStore struct { } type testHTTPInfrastructureStore struct { - records map[string]*types.HTTPIdempotencyRecord deadLetters []types.KafkaDeadLetter } @@ -104,39 +103,6 @@ func (h *testPublicMediaHandler) UploadPublic(def *media.PublicFileDef, file io. return def.Location, int64(len(data)), nil } -func (s *testHTTPInfrastructureStore) recordKey(record *types.HTTPIdempotencyRecord) string { - return record.UserID.String() + "|" + record.Method + "|" + record.Route + "|" + record.Key -} - -func (s *testHTTPInfrastructureStore) BeginIdempotency(_ types.TenantID, record *types.HTTPIdempotencyRecord) (*types.HTTPIdempotencyRecord, bool, error) { - if s.records == nil { - s.records = make(map[string]*types.HTTPIdempotencyRecord) - } - key := s.recordKey(record) - if existing := s.records[key]; existing != nil { - copy := *existing - copy.ResponseBody = append([]byte(nil), existing.ResponseBody...) - return ©, false, nil - } - copy := *record - copy.State = types.HTTPIdempotencyProcessing - s.records[key] = © - return ©, true, nil -} - -func (s *testHTTPInfrastructureStore) CompleteIdempotency(_ types.TenantID, record *types.HTTPIdempotencyRecord) error { - copy := *record - copy.State = types.HTTPIdempotencyCompleted - copy.ResponseBody = append([]byte(nil), record.ResponseBody...) - s.records[s.recordKey(record)] = © - return nil -} - -func (s *testHTTPInfrastructureStore) AbortIdempotency(_ types.TenantID, record *types.HTTPIdempotencyRecord) error { - delete(s.records, s.recordKey(record)) - return nil -} - func (s *testHTTPInfrastructureStore) BeginKafkaConsume(_ types.TenantID, record *types.KafkaConsumeRecord, _ time.Time) (*types.KafkaConsumeRecord, bool, error) { copy := *record copy.Status = types.KafkaConsumeProcessing @@ -692,8 +658,8 @@ func (s *testUserSecurityStore) CountUnreadSystemNotificationsByEventPrefix(_ ty return s.unreadByPrefix[prefix], nil } -func (s *testUserSecurityStore) MarkSystemNotificationRead(_ types.TenantID, _ types.Uid, _ string, _ time.Time) error { - return nil +func (s *testUserSecurityStore) MarkSystemNotificationRead(_ types.TenantID, _ types.Uid, eventID string, readAt time.Time) (*types.SystemNotification, error) { + return &types.SystemNotification{EventID: eventID, EventType: "system.notice", Title: "通知", Summary: "通知", ReadAt: &readAt, CreatedAt: readAt}, nil } func (s *testProfilesStore) Get(_ types.TenantID, uid types.Uid) (*types.UserProfile, error) { @@ -1229,10 +1195,45 @@ func TestIMHTTPIdempotencyReplayAndConflict(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() expectIMBearer(t, ctrl, "token-1", 7, 42) - oldInfrastructure := store.HTTPInfrastructure - infrastructure := &testHTTPInfrastructureStore{} - store.HTTPInfrastructure = infrastructure - defer func() { store.HTTPInfrastructure = oldInfrastructure }() + oldBegin, oldComplete, oldAbort := beginIMHTTPIdempotency, completeIMHTTPIdempotency, abortIMHTTPIdempotency + type cachedResponse struct { + bodyHash string + replay *imHTTPIdempotencyReplay + } + records := map[string]*cachedResponse{} + recordKey := func(tenantID types.TenantID, uid types.Uid, method, route, key string) string { + return imHTTPIdempotencyID(tenantID, uid, method, route, key) + } + beginIMHTTPIdempotency = func(tenantID types.TenantID, uid types.Uid, method, route, key, bodyHash string) (*imHTTPIdempotencyReplay, bool, bool, error) { + id := recordKey(tenantID, uid, method, route, key) + if existing := records[id]; existing != nil { + if existing.bodyHash != bodyHash { + return nil, false, true, errIMHTTPIdempotencyConflict + } + if existing.replay == nil { + return nil, false, true, nil + } + copy := *existing.replay + copy.ResponseBody = append([]byte(nil), existing.replay.ResponseBody...) + return ©, false, true, nil + } + records[id] = &cachedResponse{bodyHash: bodyHash} + return nil, true, true, nil + } + completeIMHTTPIdempotency = func(tenantID types.TenantID, uid types.Uid, method, route, key, bodyHash string, status int, body []byte) error { + records[recordKey(tenantID, uid, method, route, key)] = &cachedResponse{ + bodyHash: bodyHash, + replay: &imHTTPIdempotencyReplay{BodyHash: bodyHash, ResponseStatus: status, ResponseBody: append([]byte(nil), body...)}, + } + return nil + } + abortIMHTTPIdempotency = func(tenantID types.TenantID, uid types.Uid, method, route, key string) error { + delete(records, recordKey(tenantID, uid, method, route, key)) + return nil + } + defer func() { + beginIMHTTPIdempotency, completeIMHTTPIdempotency, abortIMHTTPIdempotency = oldBegin, oldComplete, oldAbort + }() called := 0 handler := withIMHTTPProtocol("/v1/im/", http.HandlerFunc(func(wrt http.ResponseWriter, req *http.Request) { @@ -3085,6 +3086,26 @@ func TestServeIMSystemNotificationsListAndRead(t *testing.T) { if security.readTenant != 7 || security.readUser != 42 || security.readNotificationID != "evt-1" || security.notifications[0].ReadAt == nil { t.Fatalf("notification was not marked read: %#v", security) } + readData := decodeIMHTTPResponse(t, rec).Data.(map[string]any) + if readData["id"] != "evt-1" || readData["status"] != "ok" || readData["read_at"] != security.notifications[0].ReadAt.UTC().Format(time.RFC3339Nano) { + t.Fatalf("read response did not return persisted notification state: %#v", readData) + } + + expectIMBearer(t, ctrl, "token-read-again", 7, 42) + req = httptest.NewRequest(http.MethodPost, "/v1/im/system-notifications/evt-2/read", bytes.NewBufferString(`{"read_at":"2026-09-01T00:00:00Z"}`)) + req.Header.Set("Authorization", "Bearer token-read-again") + req.Header.Set("Idempotency-Key", "read-2") + rec = httptest.NewRecorder() + + serveIMSystemNotificationRead(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("read again status: got %d, want %d; body=%s", rec.Code, http.StatusOK, rec.Body.String()) + } + readAgainData := decodeIMHTTPResponse(t, rec).Data.(map[string]any) + if readAgainData["id"] != "evt-2" || readAgainData["read_at"] != readAt.UTC().Format(time.RFC3339Nano) { + t.Fatalf("read again response should keep persisted read_at: %#v", readAgainData) + } } func TestServeIMMessageReceipts(t *testing.T) { @@ -5211,6 +5232,19 @@ func TestServeIMFavoriteBatchCreateNormalizesP2PUserTopic(t *testing.T) { } } +func TestFavoriteSourcePayloadIncludesP2PPeer(t *testing.T) { + viewer := types.Uid(42) + peer := types.Uid(43) + topic := viewer.P2PName(peer) + source := favoriteSourcePayload(topic, 17, viewer) + if source["topic"] != topic || source["seq"] != int64(17) { + t.Fatalf("favorite source should keep stored topic and seq: %#v", source) + } + if source["peer_user_id"] != peer.UserId() || source["display_topic"] != peer.UserId() { + t.Fatalf("favorite source should expose addressable p2p peer: %#v", source) + } +} + func TestServeIMFavoriteForwardSavesMessages(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() diff --git a/server/internal_admin_query_resources.go b/server/internal_admin_query_resources.go index e6e6d2d..e8526c6 100644 --- a/server/internal_admin_query_resources.go +++ b/server/internal_admin_query_resources.go @@ -68,11 +68,6 @@ var internalTenantResourceSpecs = map[string]*internalQuerySpec{ From: "FROM im_user_login_logs", DefaultSort: "ORDER BY id DESC", DetailKey: "id", Filters: []internalQueryFilter{{"user_id", "user_id", "=", "uid"}, {"account", "account_mask", "LIKE", "string"}, {"login_type", "login_type", "=", "string"}, {"event_type", "event_type", "=", "string"}, {"platform", "platform", "=", "string"}, {"ip", "ip", "LIKE", "string"}, {"login_at_start", "login_at", ">=", "time"}, {"login_at_end", "login_at", "<=", "time"}}, }, - "login-failure-logs": { - Select: `SELECT id,CAST(user_id AS CHAR) AS user_id,account_type,account_mask,account_hash,failure_type,reason,detail,ip,region,device_id,platform,app_version,user_agent,risk_level,risk_action,request_id,attempt_at,metadata`, - From: "FROM im_login_failure_logs", DefaultSort: "ORDER BY id DESC", DetailKey: "id", - Filters: []internalQueryFilter{{"user_id", "user_id", "=", "uid"}, {"account", "account_mask", "LIKE", "string"}, {"failure_type", "failure_type", "=", "string"}, {"platform", "platform", "=", "string"}, {"ip", "ip", "LIKE", "string"}, {"attempt_at_start", "attempt_at", ">=", "time"}, {"attempt_at_end", "attempt_at", "<=", "time"}}, - }, "audit-logs": { Select: `SELECT id,request_id,actor_uid AS actor_user_id,action,target_type,target_id,ip,user_agent, JSON_OBJECT('actor_role',actor_role,'before_snapshot',before_snapshot,'after_snapshot',after_snapshot,'reason',reason,'metadata',metadata) AS detail,created_at`, @@ -439,9 +434,7 @@ func internalGroupMessageSource(tenantID types.TenantID) (string, error) { selects = append(selects, `SELECT id,topic COLLATE utf8mb4_0900_ai_ci AS topic,seqid,from_user, msg_type COLLATE utf8mb4_0900_ai_ci AS msg_type,status COLLATE utf8mb4_0900_ai_ci AS status, content,head,reply_seqid,mentioned_user_ids,created_at,updated_at - FROM im_group_messages AS legacy WHERE NOT EXISTS ( - SELECT 1 FROM im_group_message_topic_shards AS route - WHERE route.topic=legacy.topic COLLATE utf8mb4_0900_ai_ci AND legacy.seqid BETWEEN route.min_seq AND route.max_seq)`) + FROM im_group_messages AS legacy`) return "(" + strings.Join(selects, " UNION ALL ") + ")", nil } diff --git a/server/store/group_message_runtime.go b/server/store/group_message_runtime.go index 8450922..187bc93 100644 --- a/server/store/group_message_runtime.go +++ b/server/store/group_message_runtime.go @@ -128,6 +128,9 @@ type groupMessageRuntime interface { AllowRate(tenantID types.TenantID, scope string, limit int, window time.Duration) (bool, int, time.Duration, error) ClaimShortState(namespace, id string, ttl time.Duration) (bool, error) ClaimShortStateValue(namespace, id, value string, ttl time.Duration) (bool, bool, error) + SetShortStateValue(namespace, id, value string, ttl time.Duration) error + GetShortStateValue(namespace, id string) (string, bool, error) + DeleteShortState(namespace, id string) error ShortStateExists(namespace, id string) (bool, error) StoreLoginSession(token []byte, session LoginSession, ttl time.Duration) error } @@ -266,6 +269,28 @@ func ClaimShortStateValue(namespace, id, value string, ttl time.Duration) (bool, return claimed, sameValue, true, err } +func SetShortStateValue(namespace, id, value string, ttl time.Duration) (bool, error) { + if groupMsgRuntime == nil || namespace == "" || id == "" || value == "" { + return false, nil + } + return true, groupMsgRuntime.SetShortStateValue(namespace, id, value, ttl) +} + +func GetShortStateValue(namespace, id string) (string, bool, bool, error) { + if groupMsgRuntime == nil || namespace == "" || id == "" { + return "", false, false, nil + } + value, found, err := groupMsgRuntime.GetShortStateValue(namespace, id) + return value, found, true, err +} + +func DeleteShortState(namespace, id string) (bool, error) { + if groupMsgRuntime == nil || namespace == "" || id == "" { + return false, nil + } + return true, groupMsgRuntime.DeleteShortState(namespace, id) +} + func RevokeLoginToken(tenantID types.TenantID, token []byte, ttl time.Duration) (bool, error) { if groupMsgRuntime == nil || !tenantID.IsValid() || len(token) == 0 || ttl <= 0 { return false, nil @@ -1272,6 +1297,28 @@ func (r *redisGroupMessageRuntime) ClaimShortStateValue(namespace, id, value str return false, existing == value, nil } +func (r *redisGroupMessageRuntime) SetShortStateValue(namespace, id, value string, ttl time.Duration) error { + if ttl <= 0 { + ttl = time.Duration(r.cfg.ShortTTL) * time.Second + } + return r.client.Set(context.Background(), r.shortStateKey(namespace, id), value, ttl).Err() +} + +func (r *redisGroupMessageRuntime) GetShortStateValue(namespace, id string) (string, bool, error) { + value, err := r.client.Get(context.Background(), r.shortStateKey(namespace, id)).Result() + if err == redis.Nil { + return "", false, nil + } + if err != nil { + return "", false, err + } + return value, true, nil +} + +func (r *redisGroupMessageRuntime) DeleteShortState(namespace, id string) error { + return r.client.Del(context.Background(), r.shortStateKey(namespace, id)).Err() +} + func (r *redisGroupMessageRuntime) ShortStateExists(namespace, id string) (bool, error) { count, err := r.client.Exists(context.Background(), r.shortStateKey(namespace, id)).Result() return count > 0, err @@ -1450,7 +1497,7 @@ func (r *redisGroupMessageRuntime) loginSessionKey(tokenHash string) string { func (r *redisGroupMessageRuntime) cacheTTL(namespace string) time.Duration { ttl := r.cfg.CacheTTL - if namespace == "verification" || namespace == "action_token" { + if namespace == "verification" || namespace == "action_token" || namespace == "login_lockout" { ttl = r.cfg.ShortTTL } return time.Duration(ttl) * time.Second diff --git a/server/store/group_message_runtime_test.go b/server/store/group_message_runtime_test.go index 4616b5b..0136d7f 100644 --- a/server/store/group_message_runtime_test.go +++ b/server/store/group_message_runtime_test.go @@ -15,6 +15,7 @@ type testGroupMessageRuntime struct { claims map[string]*groupMessageUniqueRecord seqs map[string]int64 shortStates map[string]bool + shortValues map[string]string objects map[string][]byte deletedPrefixes []string deletedObjects []string @@ -25,6 +26,7 @@ func newTestGroupMessageRuntime() *testGroupMessageRuntime { claims: make(map[string]*groupMessageUniqueRecord), seqs: make(map[string]int64), shortStates: make(map[string]bool), + shortValues: make(map[string]string), objects: make(map[string][]byte), } } @@ -209,8 +211,29 @@ func (r *testGroupMessageRuntime) ClaimShortState(namespace, id string, _ time.D return true, nil } -func (r *testGroupMessageRuntime) ClaimShortStateValue(string, string, string, time.Duration) (bool, bool, error) { - return true, true, nil +func (r *testGroupMessageRuntime) ClaimShortStateValue(namespace, id, value string, _ time.Duration) (bool, bool, error) { + key := namespace + ":" + id + if existing, ok := r.shortValues[key]; ok { + return false, existing == value, nil + } + r.shortValues[key] = value + return true, false, nil +} + +func (r *testGroupMessageRuntime) SetShortStateValue(namespace, id, value string, _ time.Duration) error { + r.shortValues[namespace+":"+id] = value + return nil +} + +func (r *testGroupMessageRuntime) GetShortStateValue(namespace, id string) (string, bool, error) { + value, ok := r.shortValues[namespace+":"+id] + return value, ok, nil +} + +func (r *testGroupMessageRuntime) DeleteShortState(namespace, id string) error { + delete(r.shortStates, namespace+":"+id) + delete(r.shortValues, namespace+":"+id) + return nil } func (r *testGroupMessageRuntime) ShortStateExists(namespace, id string) (bool, error) { diff --git a/server/store/infrastructure.go b/server/store/infrastructure.go index d0233e6..5e45394 100644 --- a/server/store/infrastructure.go +++ b/server/store/infrastructure.go @@ -1,6 +1,10 @@ package store import ( + "encoding/json" + "errors" + "strconv" + "strings" "time" adapter "github.com/tinode/chat/server/db" @@ -8,9 +12,6 @@ import ( ) type HTTPInfrastructurePersistenceInterface interface { - BeginIdempotency(types.TenantID, *types.HTTPIdempotencyRecord) (*types.HTTPIdempotencyRecord, bool, error) - CompleteIdempotency(types.TenantID, *types.HTTPIdempotencyRecord) error - AbortIdempotency(types.TenantID, *types.HTTPIdempotencyRecord) error BeginKafkaConsume(types.TenantID, *types.KafkaConsumeRecord, time.Time) (*types.KafkaConsumeRecord, bool, error) MarkKafkaConsumeSucceeded(types.TenantID, string, string, time.Time) error MarkKafkaConsumeFailed(types.TenantID, string, string, string, string, time.Time) (*types.KafkaConsumeRecord, error) @@ -25,6 +26,49 @@ type httpInfrastructureMapper struct{} var HTTPInfrastructure HTTPInfrastructurePersistenceInterface = httpInfrastructureMapper{} +const ( + kafkaConsumeNamespace = "kafka_consume" + kafkaConsumeRecordTTL = 7 * 24 * time.Hour +) + +var errKafkaConsumeRedisUnavailable = errors.New("kafka consume redis unavailable") + +func kafkaConsumeKey(tenantID types.TenantID, eventID, consumerGroup string) string { + return strconv.FormatInt(int64(tenantID), 10) + ":" + strings.TrimSpace(eventID) + ":" + strings.TrimSpace(consumerGroup) +} + +func kafkaConsumeRecordKey(tenantID types.TenantID, eventID, consumerGroup string) string { + return kafkaConsumeKey(tenantID, eventID, consumerGroup) + ":record" +} + +func kafkaConsumeRead(tenantID types.TenantID, eventID, consumerGroup string) (*types.KafkaConsumeRecord, bool, bool, error) { + raw, found, enabled, err := GetShortStateValue(kafkaConsumeNamespace, kafkaConsumeRecordKey(tenantID, eventID, consumerGroup)) + if err != nil || !enabled || !found { + return nil, found, enabled, err + } + var record types.KafkaConsumeRecord + if err = json.Unmarshal([]byte(raw), &record); err != nil { + _, _ = DeleteShortState(kafkaConsumeNamespace, kafkaConsumeRecordKey(tenantID, eventID, consumerGroup)) + return nil, false, true, err + } + return &record, true, true, nil +} + +func kafkaConsumeWrite(tenantID types.TenantID, record *types.KafkaConsumeRecord) error { + if record == nil { + return types.ErrMalformed + } + payload, err := json.Marshal(record) + if err != nil { + return err + } + enabled, err := SetShortStateValue(kafkaConsumeNamespace, kafkaConsumeRecordKey(tenantID, record.EventID, record.ConsumerGroup), string(payload), kafkaConsumeRecordTTL) + if !enabled { + return errKafkaConsumeRedisUnavailable + } + return err +} + func tenantHTTPInfrastructureAdapter(tenantID types.TenantID) (adapter.TenantHTTPInfrastructureAdapter, error) { business, err := tenantBusinessAdapter(tenantID) if err != nil { @@ -37,52 +81,109 @@ func tenantHTTPInfrastructureAdapter(tenantID types.TenantID) (adapter.TenantHTT return infrastructure, nil } -func (httpInfrastructureMapper) BeginIdempotency(tenantID types.TenantID, record *types.HTTPIdempotencyRecord) (*types.HTTPIdempotencyRecord, bool, error) { - infrastructure, err := tenantHTTPInfrastructureAdapter(tenantID) - if err != nil { - return nil, false, err - } - return infrastructure.TenantHTTPIdempotencyBegin(tenantID, record) -} - -func (httpInfrastructureMapper) CompleteIdempotency(tenantID types.TenantID, record *types.HTTPIdempotencyRecord) error { - infrastructure, err := tenantHTTPInfrastructureAdapter(tenantID) - if err != nil { - return err - } - return infrastructure.TenantHTTPIdempotencyComplete(tenantID, record) -} - -func (httpInfrastructureMapper) AbortIdempotency(tenantID types.TenantID, record *types.HTTPIdempotencyRecord) error { - infrastructure, err := tenantHTTPInfrastructureAdapter(tenantID) - if err != nil { - return err - } - return infrastructure.TenantHTTPIdempotencyAbort(tenantID, record) -} - func (httpInfrastructureMapper) BeginKafkaConsume(tenantID types.TenantID, record *types.KafkaConsumeRecord, deadline time.Time) (*types.KafkaConsumeRecord, bool, error) { - infrastructure, err := tenantHTTPInfrastructureAdapter(tenantID) - if err != nil { + if record == nil || strings.TrimSpace(record.EventID) == "" || strings.TrimSpace(record.ConsumerGroup) == "" || strings.TrimSpace(record.SourceTopic) == "" { + return nil, false, types.ErrMalformed + } + if existing, found, enabled, err := kafkaConsumeRead(tenantID, record.EventID, record.ConsumerGroup); err != nil || !enabled { + if err == nil { + err = errKafkaConsumeRedisUnavailable + } + return nil, false, err + } else if found && (existing.Status == types.KafkaConsumeSucceeded || existing.Status == types.KafkaConsumeDead) { + return existing, false, nil + } else if found { + record.RetryCount = existing.RetryCount + record.LastReason = existing.LastReason + record.LastError = existing.LastError + record.CreatedAt = existing.CreatedAt + } + ttl := time.Until(deadline) + if ttl <= 0 { + ttl = 5 * time.Minute + } + claimed, _, enabled, err := ClaimShortStateValue(kafkaConsumeNamespace, kafkaConsumeKey(tenantID, record.EventID, record.ConsumerGroup), types.KafkaConsumeProcessing, ttl) + if err != nil || !enabled { + if err == nil { + err = errKafkaConsumeRedisUnavailable + } return nil, false, err } - return infrastructure.TenantKafkaConsumeBegin(tenantID, record, deadline) + if !claimed { + existing, found, enabled, err := kafkaConsumeRead(tenantID, record.EventID, record.ConsumerGroup) + if err != nil || !enabled { + if err == nil { + err = errKafkaConsumeRedisUnavailable + } + return nil, false, err + } + if found { + return existing, false, nil + } + copy := *record + copy.Status = types.KafkaConsumeProcessing + copy.ProcessingDeadline = &deadline + return ©, false, nil + } + now := time.Now().UTC().Round(time.Millisecond) + copy := *record + copy.Status = types.KafkaConsumeProcessing + copy.ProcessingDeadline = &deadline + if copy.CreatedAt.IsZero() { + copy.CreatedAt = now + } + copy.UpdatedAt = now + if err = kafkaConsumeWrite(tenantID, ©); err != nil { + _, _ = DeleteShortState(kafkaConsumeNamespace, kafkaConsumeKey(tenantID, record.EventID, record.ConsumerGroup)) + return nil, false, err + } + return ©, true, nil } func (httpInfrastructureMapper) MarkKafkaConsumeSucceeded(tenantID types.TenantID, eventID, consumerGroup string, now time.Time) error { - infrastructure, err := tenantHTTPInfrastructureAdapter(tenantID) - if err != nil { + record, found, enabled, err := kafkaConsumeRead(tenantID, eventID, consumerGroup) + if err != nil || !enabled { + if err == nil { + err = errKafkaConsumeRedisUnavailable + } return err } - return infrastructure.TenantKafkaConsumeSucceed(tenantID, eventID, consumerGroup, now) + if !found { + return types.ErrNotFound + } + record.Status = types.KafkaConsumeSucceeded + record.SucceededAt = &now + record.ProcessingDeadline = nil + record.UpdatedAt = now + if err = kafkaConsumeWrite(tenantID, record); err != nil { + return err + } + _, err = DeleteShortState(kafkaConsumeNamespace, kafkaConsumeKey(tenantID, eventID, consumerGroup)) + return err } func (httpInfrastructureMapper) MarkKafkaConsumeFailed(tenantID types.TenantID, eventID, consumerGroup, reason, message string, now time.Time) (*types.KafkaConsumeRecord, error) { - infrastructure, err := tenantHTTPInfrastructureAdapter(tenantID) - if err != nil { + record, found, enabled, err := kafkaConsumeRead(tenantID, eventID, consumerGroup) + if err != nil || !enabled { + if err == nil { + err = errKafkaConsumeRedisUnavailable + } return nil, err } - return infrastructure.TenantKafkaConsumeFail(tenantID, eventID, consumerGroup, reason, message, now) + if !found { + return nil, types.ErrNotFound + } + record.Status = types.KafkaConsumeFailed + record.RetryCount++ + record.LastReason = reason + record.LastError = message + record.ProcessingDeadline = nil + record.UpdatedAt = now + if err = kafkaConsumeWrite(tenantID, record); err != nil { + return nil, err + } + _, err = DeleteShortState(kafkaConsumeNamespace, kafkaConsumeKey(tenantID, eventID, consumerGroup)) + return record, err } func (httpInfrastructureMapper) MarkKafkaConsumeDead(tenantID types.TenantID, dead *types.KafkaDeadLetter, now time.Time) error { @@ -90,7 +191,18 @@ func (httpInfrastructureMapper) MarkKafkaConsumeDead(tenantID types.TenantID, de if err != nil { return err } - return infrastructure.TenantKafkaConsumeDead(tenantID, dead, now) + if err = infrastructure.TenantKafkaConsumeDead(tenantID, dead, now); err != nil { + return err + } + record := &types.KafkaConsumeRecord{ + EventID: dead.EventID, ConsumerGroup: dead.ConsumerGroup, SourceTopic: dead.SourceTopic, + EventType: dead.EventType, BizType: dead.BizType, BizID: dead.BizID, + Status: types.KafkaConsumeDead, RetryCount: dead.RetryCount, LastReason: dead.LastReason, + LastError: dead.LastError, DeadAt: &now, CreatedAt: now, UpdatedAt: now, + } + _ = kafkaConsumeWrite(tenantID, record) + _, _ = DeleteShortState(kafkaConsumeNamespace, kafkaConsumeKey(tenantID, dead.EventID, dead.ConsumerGroup)) + return nil } func (httpInfrastructureMapper) ListKafkaDeadLetters(tenantID types.TenantID, opts types.KafkaDeadLetterListOptions) (*types.KafkaDeadLetterPage, error) { @@ -122,5 +234,11 @@ func (httpInfrastructureMapper) MarkKafkaDeadLetterReplay(tenantID types.TenantI if err != nil { return nil, err } - return infrastructure.TenantKafkaDeadLetterMarkReplay(tenantID, deadID, resolvedBy, note, now) + dead, err := infrastructure.TenantKafkaDeadLetterMarkReplay(tenantID, deadID, resolvedBy, note, now) + if err != nil { + return nil, err + } + _, _ = DeleteShortState(kafkaConsumeNamespace, kafkaConsumeKey(tenantID, dead.EventID, dead.ConsumerGroup)) + _, _ = DeleteShortState(kafkaConsumeNamespace, kafkaConsumeRecordKey(tenantID, dead.EventID, dead.ConsumerGroup)) + return dead, nil } diff --git a/server/store/store.go b/server/store/store.go index a6d2055..93fc923 100644 --- a/server/store/store.go +++ b/server/store/store.go @@ -3,6 +3,7 @@ package store import ( "context" + "crypto/sha256" "encoding/json" "errors" "fmt" @@ -4326,7 +4327,7 @@ type UserSecurityPersistenceInterface interface { ListSystemNotifications(tenantID types.TenantID, uid types.Uid, cursor string, limit int) ([]types.SystemNotification, string, error) CountUnreadSystemNotifications(tenantID types.TenantID, uid types.Uid) (int, error) CountUnreadSystemNotificationsByEventPrefix(tenantID types.TenantID, uid types.Uid, eventPrefix string) (int, error) - MarkSystemNotificationRead(tenantID types.TenantID, uid types.Uid, eventID string, readAt time.Time) error + MarkSystemNotificationRead(tenantID types.TenantID, uid types.Uid, eventID string, readAt time.Time) (*types.SystemNotification, error) } type userSecurityMapper struct{} @@ -4384,30 +4385,39 @@ func (userSecurityMapper) CreateActionToken(tenantID types.TenantID, token *type if token == nil || token.TenantID != tenantID { return types.ErrMalformed } - tenantAdp, err := tenantBusinessAdapter(tenantID) - if err != nil { - return err - } - if err = tenantAdp.TenantActionTokenCreate(tenantID, token); err != nil { - return err - } - if cacheErr := CacheObject(tenantID, cacheNamespaceActionToken, actionTokenCacheID(token.UserID, token.TokenID, token.Action, token.Target), token); cacheErr != nil { - logs.Warn.Printf("cache action token[%s]: write failed: %v", token.TokenID, cacheErr) - } - return nil + now := types.TimeNow() + token.CreatedAt = now + token.UpdatedAt = now + return CacheObject(tenantID, cacheNamespaceActionToken, actionTokenCacheID(token.UserID, token.TokenID, token.Action, token.Target), token) } func (userSecurityMapper) ConsumeActionToken(tenantID types.TenantID, uid types.Uid, token, action, target string, now time.Time) (*types.ActionToken, error) { - tenantAdp, err := tenantBusinessAdapter(tenantID) + if !tenantID.IsValid() || uid.IsZero() || token == "" || action == "" { + return nil, types.ErrMalformed + } + if now.IsZero() { + now = types.TimeNow() + } + tokenID, secret, ok := strings.Cut(token, ".") + if !ok || tokenID == "" || secret == "" { + return nil, types.ErrMalformed + } + cacheID := actionTokenCacheID(uid, tokenID, action, target) + var cached types.ActionToken + ok, err := GetCachedObject(tenantID, cacheNamespaceActionToken, cacheID, &cached) if err != nil { return nil, err } - consumed, err := tenantAdp.TenantActionTokenConsume(tenantID, uid, token, action, target, now) - if err == nil { - tokenID, _, _ := strings.Cut(token, ".") - _ = DeleteCachedObject(tenantID, cacheNamespaceActionToken, actionTokenCacheID(uid, tokenID, action, target)) + if !ok || cached.State != 0 || cached.TokenHash != actionTokenHash(tenantID, uid, action, target, tokenID, secret) || !now.Before(cached.ExpiresAt) { + return nil, types.ErrNotFound } - return consumed, err + cached.State = 1 + cached.ConsumedAt = &now + cached.UpdatedAt = now + if err = DeleteCachedObject(tenantID, cacheNamespaceActionToken, cacheID); err != nil { + return nil, err + } + return &cached, nil } func (userSecurityMapper) GetIndependentPassword(tenantID types.TenantID, uid types.Uid, purpose string) (*types.IndependentPassword, error) { @@ -4500,10 +4510,10 @@ func (userSecurityMapper) CountUnreadSystemNotificationsByEventPrefix(tenantID t return tenantAdp.TenantSystemNotificationUnreadCountByEventPrefix(tenantID, uid, eventPrefix) } -func (userSecurityMapper) MarkSystemNotificationRead(tenantID types.TenantID, uid types.Uid, eventID string, readAt time.Time) error { +func (userSecurityMapper) MarkSystemNotificationRead(tenantID types.TenantID, uid types.Uid, eventID string, readAt time.Time) (*types.SystemNotification, error) { tenantAdp, err := tenantBusinessAdapter(tenantID) if err != nil { - return err + return nil, err } return tenantAdp.TenantSystemNotificationMarkRead(tenantID, uid, eventID, readAt) } @@ -4512,26 +4522,30 @@ func (verificationMapper) Create(tenantID types.TenantID, vc *types.Verification if vc == nil || vc.TenantID != tenantID { return types.ErrMalformed } + if cacheErr := cacheVerificationCode(tenantID, vc); cacheErr != nil { + return cacheErr + } tenantAdp, err := tenantBusinessAdapter(tenantID) if err != nil { - return err + logs.Warn.Printf("verification[%s]: audit adapter unavailable: %v", vc.ChallengeID, err) + return nil } if err = tenantAdp.TenantVerificationCreate(tenantID, vc); err != nil { - return err - } - if cacheErr := cacheVerificationCode(tenantID, vc); cacheErr != nil { - logs.Warn.Printf("cache verification[%s]: write failed: %v", vc.ChallengeID, cacheErr) + logs.Warn.Printf("verification[%s]: audit insert failed: %v", vc.ChallengeID, err) } return nil } func (verificationMapper) Get(tenantID types.TenantID, challengeID string) (*types.VerificationCode, error) { - _ = DeleteCachedObject(tenantID, cacheNamespaceVerification, challengeID) - tenantAdp, err := tenantBusinessAdapter(tenantID) + var vc types.VerificationCode + ok, err := GetCachedObject(tenantID, cacheNamespaceVerification, challengeID, &vc) if err != nil { return nil, err } - return tenantAdp.TenantVerificationGet(tenantID, challengeID) + if !ok { + return nil, types.ErrNotFound + } + return &vc, nil } func (verificationMapper) GetActive(tenantID types.TenantID, scene, phoneHash string, now time.Time) (*types.VerificationCode, error) { @@ -4547,74 +4561,59 @@ func (verificationMapper) GetActive(tenantID types.TenantID, scene, phoneHash st } _ = DeleteCachedObject(tenantID, cacheNamespaceVerification, activeCacheID) } - tenantAdp, err := tenantBusinessAdapter(tenantID) - if err != nil { - return nil, err - } - vc, err := tenantAdp.TenantVerificationGetActive(tenantID, scene, phoneHash, now) - if err == nil && vc != nil { - if cacheErr := cacheVerificationCode(tenantID, vc); cacheErr != nil { - logs.Warn.Printf("cache verification active[%s]: write failed: %v", activeCacheID, cacheErr) - } - } - return vc, err + return nil, types.ErrNotFound } func (verificationMapper) Fail(tenantID types.TenantID, challengeID string, state types.VerificationState) error { - tenantAdp, err := tenantBusinessAdapter(tenantID) + vc, err := Verifications.Get(tenantID, challengeID) if err != nil { return err } - if err = tenantAdp.TenantVerificationFail(tenantID, challengeID, state); err != nil { - return err - } - if vc, getErr := tenantAdp.TenantVerificationGet(tenantID, challengeID); getErr == nil && vc != nil { - _ = cacheVerificationCode(tenantID, vc) - } else { - _ = DeleteCachedObject(tenantID, cacheNamespaceVerification, challengeID) - } - return nil + vc.Attempts++ + vc.State = state + vc.UpdatedAt = types.TimeNow() + return cacheVerificationCode(tenantID, vc) } func (verificationMapper) MarkVerified(tenantID types.TenantID, challengeID, tokenHash string, verifiedAt time.Time) error { - tenantAdp, err := tenantBusinessAdapter(tenantID) + vc, err := Verifications.Get(tenantID, challengeID) if err != nil { return err } - if err = tenantAdp.TenantVerificationMarkVerified(tenantID, challengeID, tokenHash, verifiedAt); err != nil { - return err + if vc.State != types.VerificationStatePending || !verifiedAt.Before(vc.ExpiresAt) { + return types.ErrNotFound } - if vc, getErr := tenantAdp.TenantVerificationGet(tenantID, challengeID); getErr == nil && vc != nil { - _ = cacheVerificationCode(tenantID, vc) - _ = DeleteCachedObject(tenantID, cacheNamespaceVerification, verificationActiveCacheID(vc.Scene, vc.PhoneHash)) - } else { - _ = DeleteCachedObject(tenantID, cacheNamespaceVerification, challengeID) - } - return nil + vc.State = types.VerificationStateVerified + vc.VerificationTokenHash = &tokenHash + vc.VerifiedAt = &verifiedAt + vc.UpdatedAt = verifiedAt + return cacheVerificationCode(tenantID, vc) } func (verificationMapper) Consume(tenantID types.TenantID, challengeID, tokenHash string, consumedAt time.Time) error { - tenantAdp, err := tenantBusinessAdapter(tenantID) + vc, err := Verifications.Get(tenantID, challengeID) if err != nil { return err } - if err = tenantAdp.TenantVerificationConsume(tenantID, challengeID, tokenHash, consumedAt); err != nil { + if vc.State != types.VerificationStateVerified || vc.VerificationTokenHash == nil || *vc.VerificationTokenHash != tokenHash || !consumedAt.Before(vc.ExpiresAt) { + return types.ErrNotFound + } + vc.State = types.VerificationStateConsumed + vc.ConsumedAt = &consumedAt + vc.UpdatedAt = consumedAt + if err = cacheVerificationCode(tenantID, vc); err != nil { return err } - if vc, getErr := tenantAdp.TenantVerificationGet(tenantID, challengeID); getErr == nil && vc != nil { - _ = cacheVerificationCode(tenantID, vc) - _ = DeleteCachedObject(tenantID, cacheNamespaceVerification, verificationActiveCacheID(vc.Scene, vc.PhoneHash)) - } else { - _ = DeleteCachedObject(tenantID, cacheNamespaceVerification, challengeID) - } - return nil + return DeleteCachedObject(tenantID, cacheNamespaceVerification, verificationActiveCacheID(vc.Scene, vc.PhoneHash)) } func cacheVerificationCode(tenantID types.TenantID, vc *types.VerificationCode) error { if vc == nil { return nil } - _ = DeleteCachedObject(tenantID, cacheNamespaceVerification, vc.ChallengeID) + if err := CacheObject(tenantID, cacheNamespaceVerification, vc.ChallengeID, vc); err != nil { + return err + } if vc.State == types.VerificationStatePending { return CacheObject(tenantID, cacheNamespaceVerification, verificationActiveCacheID(vc.Scene, vc.PhoneHash), vc.ChallengeID) } @@ -4630,6 +4629,22 @@ func actionTokenCacheID(uid types.Uid, tokenID, action, target string) string { return uid.String() + ":" + strings.TrimSpace(tokenID) + ":" + strings.TrimSpace(action) + ":" + strings.TrimSpace(target) } +func actionTokenHash(tenantID types.TenantID, uid types.Uid, action, target, tokenID, secret string) string { + h := sha256.New() + h.Write([]byte(strconv.FormatInt(int64(tenantID), 10))) + h.Write([]byte{0}) + h.Write([]byte(strconv.FormatInt(DecodeUid(uid), 10))) + h.Write([]byte{0}) + h.Write([]byte(action)) + h.Write([]byte{0}) + h.Write([]byte(target)) + h.Write([]byte{0}) + h.Write([]byte(tokenID)) + h.Write([]byte{0}) + h.Write([]byte(secret)) + return fmt.Sprintf("%x", h.Sum(nil)) +} + // LoginLockoutPersistenceInterface stores per-account login failure counters // and temporary lock state in each tenant business database. type LoginLockoutPersistenceInterface interface { @@ -4665,43 +4680,71 @@ func tenantLoginLockoutAdapter(tenantID types.TenantID) (adapter.TenantLoginLock } func (loginLockoutMapper) Get(tenantID types.TenantID, accountHash string) (*types.LoginLockout, error) { - lockAdp, err := tenantLoginLockoutAdapter(tenantID) + var lockout types.LoginLockout + ok, err := GetCachedObject(tenantID, "login_lockout", accountHash, &lockout) if err != nil { return nil, err } - return lockAdp.TenantLoginLockoutGet(tenantID, accountHash) + if !ok { + return nil, types.ErrNotFound + } + return &lockout, nil } func (loginLockoutMapper) RecordFailure(tenantID types.TenantID, accountHash string, userID *int64, windowCutoff, now time.Time) (*types.LoginLockout, error) { - lockAdp, err := tenantLoginLockoutAdapter(tenantID) - if err != nil { + current, err := LoginLockouts.Get(tenantID, accountHash) + if err != nil && !errors.Is(err, types.ErrNotFound) { return nil, err } - return lockAdp.TenantLoginLockoutRecordFailure(tenantID, accountHash, userID, windowCutoff, now) + if current == nil { + current = &types.LoginLockout{TenantID: tenantID, AccountHash: accountHash} + } + if current.WindowStartAt == nil || current.WindowStartAt.Before(windowCutoff) { + current.FailedAttempts = 0 + start := now + current.WindowStartAt = &start + } + current.FailedAttempts++ + if userID != nil { + current.UserID = *userID + } + current.LastAttemptAt = &now + current.UpdatedAt = &now + return current, CacheObject(tenantID, "login_lockout", accountHash, current) } func (loginLockoutMapper) Lock(tenantID types.TenantID, accountHash string, lockedUntil, now time.Time) error { - lockAdp, err := tenantLoginLockoutAdapter(tenantID) - if err != nil { + current, err := LoginLockouts.Get(tenantID, accountHash) + if err != nil && !errors.Is(err, types.ErrNotFound) { return err } - return lockAdp.TenantLoginLockoutLock(tenantID, accountHash, lockedUntil, now) + if current == nil { + current = &types.LoginLockout{TenantID: tenantID, AccountHash: accountHash} + } + if current.LockedUntil == nil || current.LockedUntil.Before(lockedUntil) { + current.LockedUntil = &lockedUntil + } + current.UpdatedAt = &now + return CacheObject(tenantID, "login_lockout", accountHash, current) } func (loginLockoutMapper) Reset(tenantID types.TenantID, accountHash string) error { - lockAdp, err := tenantLoginLockoutAdapter(tenantID) - if err != nil { - return err - } - return lockAdp.TenantLoginLockoutReset(tenantID, accountHash) + return DeleteCachedObject(tenantID, "login_lockout", accountHash) } func (loginLockoutMapper) Release(tenantID types.TenantID, accountHash string, now time.Time) error { - lockAdp, err := tenantLoginLockoutAdapter(tenantID) - if err != nil { + current, err := LoginLockouts.Get(tenantID, accountHash) + if err != nil && !errors.Is(err, types.ErrNotFound) { return err } - return lockAdp.TenantLoginLockoutRelease(tenantID, accountHash, now) + if current == nil { + return nil + } + current.FailedAttempts = 0 + current.WindowStartAt = nil + current.LockedUntil = nil + current.UpdatedAt = &now + return CacheObject(tenantID, "login_lockout", accountHash, current) } func tenantRegistrationInviteAdapter(tenantID types.TenantID) (adapter.TenantRegistrationInviteAdapter, error) { diff --git a/server/store/types/infrastructure.go b/server/store/types/infrastructure.go index 5d97649..fbf6734 100644 --- a/server/store/types/infrastructure.go +++ b/server/store/types/infrastructure.go @@ -7,9 +7,6 @@ import ( ) const ( - HTTPIdempotencyProcessing = "processing" - HTTPIdempotencyCompleted = "completed" - KafkaConsumeProcessing = "processing" KafkaConsumeSucceeded = "succeeded" KafkaConsumeFailed = "failed" @@ -22,21 +19,6 @@ const ( KafkaDeadStatusResolved = "resolved" ) -// HTTPIdempotencyRecord stores the first response for one authenticated write. -type HTTPIdempotencyRecord struct { - UserID Uid - Method string - Route string - Key string - BodyHash string - State string - ResponseStatus int - ResponseBody []byte - CreatedAt time.Time - UpdatedAt time.Time - ExpiresAt time.Time -} - // KafkaConsumeRecord stores idempotent consumer state for one event and group. type KafkaConsumeRecord struct { EventID string diff --git a/server/store/types/login_lockout.go b/server/store/types/login_lockout.go index d653a5b..e131796 100644 --- a/server/store/types/login_lockout.go +++ b/server/store/types/login_lockout.go @@ -2,8 +2,8 @@ package types import "time" -// LoginLockout is the per-account login failure counter and lock state -// stored in each tenant business database (table im_login_lockouts). +// LoginLockout is the per-account login failure counter and lock state stored +// in Redis. type LoginLockout struct { TenantID TenantID `json:"tenant_id"` ID int64 `json:"id"` diff --git a/server/user_verification_test.go b/server/user_verification_test.go index aa20e01..2771132 100644 --- a/server/user_verification_test.go +++ b/server/user_verification_test.go @@ -461,18 +461,20 @@ func (s *testSecurityStore) CountUnreadSystemNotificationsByEventPrefix(tenantID return count, s.err } -func (s *testSecurityStore) MarkSystemNotificationRead(tenantID types.TenantID, uid types.Uid, eventID string, readAt time.Time) error { +func (s *testSecurityStore) MarkSystemNotificationRead(tenantID types.TenantID, uid types.Uid, eventID string, readAt time.Time) (*types.SystemNotification, error) { s.readTenant = tenantID s.readUser = uid s.readNotificationID = eventID for i := range s.notifications { if s.notifications[i].EventID == eventID { - s.notifications[i].ReadAt = &readAt - return s.err + if s.notifications[i].ReadAt == nil { + s.notifications[i].ReadAt = &readAt + } + return &s.notifications[i], s.err } } if s.err != nil { - return s.err + return nil, s.err } - return types.ErrNotFound + return nil, types.ErrNotFound }