feat: add kafka admin integration

This commit is contained in:
richjohnrambo
2026-08-28 00:05:54 +08:00
parent b2e5097708
commit 5cc6eebc1a
48 changed files with 8065 additions and 55 deletions
+11
View File
@@ -91,6 +91,17 @@ QUIC传输技术
重中之重。加密 + 接口鉴权 防止被扫 日志要记录下来 每个小时告警
法相 (三方IM 资金电话确认), [Aug 27, 2026 at 22:13:24]:
....进群的时候拿群成员列表信息 这个怎么做成缓存 然后怎么更新缓存
2.多标签要独立的登录状态,可以用不同域名来解决这个问题,但是要token过期时间要限制一下 7天或者10天 每次登录都重置这个时间
3.转发消息 涉及到转发群和转发个人 大批量转发 消息会不会串
4.我在第一个标签转发了100个群,然后直接切换到第二个标签,你第一个标签还没转发完 你该怎么处理 会不会串到第二个标签页
群成员信息这个是很耗性能的 一个群几千个人 你要考虑是进一次就拿一次群成员信息 还是拿一次
我说的进不是给你拉进群 是你点击群聊这个会话框
+34
View File
@@ -68,6 +68,40 @@ ENV REDIS_PRESENCE_TTL_SECONDS=120
ENV REDIS_CACHE_TTL_SECONDS=300
ENV REDIS_SHORT_STATE_TTL_SECONDS=600
ENV REDIS_GROUP_FANOUT_ENABLED=true
ENV KAFKA_ENABLED=false
ENV KAFKA_BROKERS_JSON='[]'
ENV KAFKA_CLIENT_ID=im-server
ENV KAFKA_CONSUMER_GROUP=im-server
ENV KAFKA_DIAL_TIMEOUT_MS=3000
ENV KAFKA_WRITE_TIMEOUT_MS=3000
ENV KAFKA_READ_TIMEOUT_MS=3000
ENV KAFKA_REQUIRED_ACKS=1
ENV KAFKA_ALLOW_AUTO_TOPIC_CREATION=false
ENV KAFKA_TLS_ENABLED=false
ENV KAFKA_MAX_RETRY_COUNT=5
ENV KAFKA_RETRY_BACKOFF_MS='[10000,30000,60000,300000,900000]'
ENV KAFKA_DEAD_LETTER_RETENTION_SECONDS=7776000
ENV KAFKA_TOPIC_ADMIN_COMMAND=im.admin.command
ENV KAFKA_TOPIC_ADMIN_COMMAND_RESULT=im.admin.command.result
ENV KAFKA_TOPIC_CONFIG_EVENT=im.config.event
ENV KAFKA_TOPIC_BUSINESS_EVENT=im.business.event
ENV KAFKA_TOPIC_MESSAGE_PERSIST=im.message.persist
ENV KAFKA_TOPIC_MESSAGE_PUSH=im.message.push
ENV KAFKA_TOPIC_FILE_SECURITY=im.file.security
ENV KAFKA_TOPIC_FILE_LIFECYCLE=im.file.lifecycle
ENV KAFKA_TOPIC_WORKBENCH_EVENT=im.workbench.event
ENV KAFKA_TOPIC_CACHE_INVALIDATE=im.cache.invalidate
ENV KAFKA_TOPIC_AUDIT_EVENT=im.audit.event
ENV KAFKA_TOPIC_DEAD_LETTER=im.dead.letter
ENV INTERNAL_API_ENABLED=false
ENV INTERNAL_API_TIME_WINDOW_SECONDS=300
ENV INTERNAL_API_HOTGO_SERVICE_NAME=hotgo-admin
ENV INTERNAL_API_HOTGO_SECRET=
ENV ADMIN_INTERNAL_API_ENABLED=false
ENV ADMIN_INTERNAL_API_BASE_URL=
ENV ADMIN_INTERNAL_API_SERVICE_NAME=im-server
ENV ADMIN_INTERNAL_API_SECRET=
ENV ADMIN_INTERNAL_API_TIMEOUT_MS=3000
# The PostgreSQL DSN connection.
ENV POSTGRES_DSN='postgresql://postgres:postgres@localhost:5432/tinode?sslmode=disable&connect_timeout=10'
+34
View File
@@ -48,6 +48,40 @@ ENV REDIS_PRESENCE_TTL_SECONDS=120
ENV REDIS_CACHE_TTL_SECONDS=300
ENV REDIS_SHORT_STATE_TTL_SECONDS=600
ENV REDIS_GROUP_FANOUT_ENABLED=true
ENV KAFKA_ENABLED=false
ENV KAFKA_BROKERS_JSON='[]'
ENV KAFKA_CLIENT_ID=im-server
ENV KAFKA_CONSUMER_GROUP=im-server
ENV KAFKA_DIAL_TIMEOUT_MS=3000
ENV KAFKA_WRITE_TIMEOUT_MS=3000
ENV KAFKA_READ_TIMEOUT_MS=3000
ENV KAFKA_REQUIRED_ACKS=1
ENV KAFKA_ALLOW_AUTO_TOPIC_CREATION=false
ENV KAFKA_TLS_ENABLED=false
ENV KAFKA_MAX_RETRY_COUNT=5
ENV KAFKA_RETRY_BACKOFF_MS='[10000,30000,60000,300000,900000]'
ENV KAFKA_DEAD_LETTER_RETENTION_SECONDS=7776000
ENV KAFKA_TOPIC_ADMIN_COMMAND=im.admin.command
ENV KAFKA_TOPIC_ADMIN_COMMAND_RESULT=im.admin.command.result
ENV KAFKA_TOPIC_CONFIG_EVENT=im.config.event
ENV KAFKA_TOPIC_BUSINESS_EVENT=im.business.event
ENV KAFKA_TOPIC_MESSAGE_PERSIST=im.message.persist
ENV KAFKA_TOPIC_MESSAGE_PUSH=im.message.push
ENV KAFKA_TOPIC_FILE_SECURITY=im.file.security
ENV KAFKA_TOPIC_FILE_LIFECYCLE=im.file.lifecycle
ENV KAFKA_TOPIC_WORKBENCH_EVENT=im.workbench.event
ENV KAFKA_TOPIC_CACHE_INVALIDATE=im.cache.invalidate
ENV KAFKA_TOPIC_AUDIT_EVENT=im.audit.event
ENV KAFKA_TOPIC_DEAD_LETTER=im.dead.letter
ENV INTERNAL_API_ENABLED=false
ENV INTERNAL_API_TIME_WINDOW_SECONDS=300
ENV INTERNAL_API_HOTGO_SERVICE_NAME=hotgo-admin
ENV INTERNAL_API_HOTGO_SECRET=
ENV ADMIN_INTERNAL_API_ENABLED=false
ENV ADMIN_INTERNAL_API_BASE_URL=
ENV ADMIN_INTERNAL_API_SERVICE_NAME=im-server
ENV ADMIN_INTERNAL_API_SECRET=
ENV ADMIN_INTERNAL_API_TIMEOUT_MS=3000
# Tenant registry lives in the hotgo database (shared with im_manager); tenant
# DB routing resolves to the same tinode DB for the seeded "default" tenant.
+47
View File
@@ -117,6 +117,53 @@
}
},
"kafka": {
"enabled": $KAFKA_ENABLED,
"brokers": $KAFKA_BROKERS_JSON,
"client_id": "$KAFKA_CLIENT_ID",
"consumer_group": "$KAFKA_CONSUMER_GROUP",
"dial_timeout_ms": $KAFKA_DIAL_TIMEOUT_MS,
"write_timeout_ms": $KAFKA_WRITE_TIMEOUT_MS,
"read_timeout_ms": $KAFKA_READ_TIMEOUT_MS,
"required_acks": $KAFKA_REQUIRED_ACKS,
"allow_auto_topic_creation": $KAFKA_ALLOW_AUTO_TOPIC_CREATION,
"tls_enabled": $KAFKA_TLS_ENABLED,
"max_retry_count": $KAFKA_MAX_RETRY_COUNT,
"retry_backoff_ms": $KAFKA_RETRY_BACKOFF_MS,
"dead_letter_retention_seconds": $KAFKA_DEAD_LETTER_RETENTION_SECONDS,
"topics": {
"admin_command": "$KAFKA_TOPIC_ADMIN_COMMAND",
"admin_command_result": "$KAFKA_TOPIC_ADMIN_COMMAND_RESULT",
"config_event": "$KAFKA_TOPIC_CONFIG_EVENT",
"business_event": "$KAFKA_TOPIC_BUSINESS_EVENT",
"message_persist": "$KAFKA_TOPIC_MESSAGE_PERSIST",
"message_push": "$KAFKA_TOPIC_MESSAGE_PUSH",
"file_security": "$KAFKA_TOPIC_FILE_SECURITY",
"file_lifecycle": "$KAFKA_TOPIC_FILE_LIFECYCLE",
"workbench_event": "$KAFKA_TOPIC_WORKBENCH_EVENT",
"cache_invalidate": "$KAFKA_TOPIC_CACHE_INVALIDATE",
"audit_event": "$KAFKA_TOPIC_AUDIT_EVENT",
"dead_letter": "$KAFKA_TOPIC_DEAD_LETTER"
}
},
"internal_api": {
"enabled": $INTERNAL_API_ENABLED,
"time_window_seconds": $INTERNAL_API_TIME_WINDOW_SECONDS,
"services": {
"$INTERNAL_API_HOTGO_SERVICE_NAME": {
"secret": "$INTERNAL_API_HOTGO_SECRET",
"allow": ["admin_query"]
}
}
},
"admin_internal_api": {
"enabled": $ADMIN_INTERNAL_API_ENABLED,
"base_url": "$ADMIN_INTERNAL_API_BASE_URL",
"service_name": "$ADMIN_INTERNAL_API_SERVICE_NAME",
"secret": "$ADMIN_INTERNAL_API_SECRET",
"timeout_ms": $ADMIN_INTERNAL_API_TIMEOUT_MS
},
"acc_validation": {
"email": {
"add_to_tags": true,
@@ -0,0 +1,599 @@
# IM Admin 内部查询接口缺口清单
> 目标:梳理 HotGo/Admin 迁移到“Admin 只访问 hotgo,IM Server 只访问租户库”后,IM Server 需要补齐的 `/internal/admin-query/*` 只读接口。
> 当前日期:2026-08-27
> 适用项目:`im_manager` Admin 后台、`im_server` IM 服务端。
## 1. 背景
Kafka 写链路已经在 Admin 侧补齐:Admin 写操作落 `im_admin_commands`、`im_admin_outbox`,通过 Kafka 发给 IM Server,IM Server 执行业务后回传 `im.admin.command.result`。
查询链路还没有完全隔离。Admin 当前仍有不少页面通过 `imtenant.DB()` 或 `resolveImTenantDB()` 直连租户库。若要严格满足目标架构,Admin 查询租户业务数据时应调用 IM Server 的内部只读接口:
```text
GET /internal/admin-query/tenants/{tenant_code}/...
```
IM Server 当前已经具备部分内部查询接口,但还不能覆盖 Admin 后台页面。
## 2. 已有接口
以下接口在 IM Server 当前实现中已经存在,可供 Admin 先迁移:
| 接口 | 用途 | 备注 |
|---|---|---|
| `GET /internal/admin-query/tenants/{tenant_code}/users/{user_id}` | 用户详情 | 需要增强字段,见 P0。 |
| `GET /internal/admin-query/tenants/{tenant_code}/groups/{topic}` | 群详情和治理配置 | 需要增强 topic 基础字段,见 P0。 |
| `GET /internal/admin-query/tenants/{tenant_code}/groups/{topic}/members` | 群成员列表 | 当前偏 cursor 分页,Admin 表格还需要 total。 |
| `GET /internal/admin-query/tenants/{tenant_code}/messages/{topic}/{seq_id}` | 消息详情 | 当前 Admin 暂未完整迁移消息页面。 |
| `GET /internal/admin-query/tenants/{tenant_code}/files/{file_id}` | 文件详情 | 还缺文件列表和引用列表。 |
| `GET /internal/admin-query/tenants/{tenant_code}/kafka/dead-letters` | Kafka 死信列表 | Admin 已接入。 |
| `GET /internal/admin-query/tenants/{tenant_code}/kafka/dead-letters/{dead_id}` | Kafka 死信详情 | Admin 已接入。 |
## 3. 统一约定
### 3.1 鉴权
沿用 IM Server 内部 API HMAC 鉴权:
```text
X-Service-Name
X-Timestamp
X-Nonce
X-Signature
```
服务端需要校验:
1. service 是否存在。
2. service 是否包含 `admin_query` capability。
3. timestamp 是否在允许窗口内。
4. nonce 是否防重放。
5. 签名是否匹配。
### 3.2 分页返回
Admin 的 `BasicTable` 更适合 page/per_page/total 格式。建议所有列表接口统一返回:
```json
{
"code": 0,
"data": {
"items": [],
"total": 0,
"page": 1,
"per_page": 20
}
}
```
如果 IM Server 内部更适合 cursor,也建议同时返回:
```json
{
"code": 0,
"data": {
"items": [],
"total": 0,
"page": 1,
"per_page": 20,
"next_cursor": "",
"has_more": false
}
}
```
### 3.3 详情返回
详情接口统一返回:
```json
{
"code": 0,
"data": {}
}
```
### 3.4 时间与 ID
1. 时间统一返回 RFC3339/RFC3339Nano 字符串。
2. 雪花 ID、Tinode 用户 ID、19 位整数 ID 建议以字符串返回,避免前端丢精度。
3. 入参同时兼容字符串 ID 和纯数字 ID 时,IM Server 内部自行解析。
## 4. P0:优先补齐
P0 是 Admin 当前核心页面迁移所需接口,补完后可以先迁移用户、群组、文件等高频页面。
### 4.1 用户列表
```text
GET /internal/admin-query/tenants/{tenant_code}/users
```
用途:IM 用户管理列表。
建议查询参数:
| 参数 | 类型 | 说明 |
|---|---|---|
| `user_id` | string | 用户 ID,支持精确查询。 |
| `nickname` | string | 昵称模糊查询。 |
| `auth_name` | string | 登录标识,匹配 username/phone/email/凭证。 |
| `state` | int | 用户状态。 |
| `created_at_start` | string | 创建时间开始。 |
| `created_at_end` | string | 创建时间结束。 |
| `last_seen_start` | string | 最后在线开始。 |
| `last_seen_end` | string | 最后在线结束。 |
| `page` | int | 页码。 |
| `per_page` | int | 每页数量,最大建议 200。 |
建议返回字段:
| 字段 | 说明 |
|---|---|
| `user_id` | 字符串用户 ID。 |
| `username` | 用户名。 |
| `phone` | 手机号。 |
| `nickname` | 昵称。 |
| `public_account` | 公开账号。 |
| `state` | 用户状态。 |
| `gender` | 性别。 |
| `region` | 地区。 |
| `auth_names` | 登录标识摘要。 |
| `credentials` | 凭证摘要。 |
| `auth_version` | 鉴权版本。 |
| `password_updated_at` | 密码更新时间。 |
| `last_seen` | 最后在线时间。 |
| `user_agent` | 最后设备 UA。 |
| `created_at` | 创建时间。 |
| `updated_at` | 更新时间。 |
当前 Admin 直连涉及表:`im_users`、`im_user_profiles`、`im_user_security`、`im_credentials`。
### 4.2 用户详情增强
现有接口:
```text
GET /internal/admin-query/tenants/{tenant_code}/users/{user_id}
```
建议增强返回字段:
| 字段 | 说明 |
|---|---|
| `profile` | 结构化用户资料。 |
| `public` | Tinode public JSON。 |
| `trusted` | Tinode trusted JSON。 |
| `access` | 访问权限 JSON。 |
| `tags` | 标签。 |
| `security` | `auth_version/password_updated_at` 等安全信息。 |
| `credentials` | 凭证列表,包含 method/value/done。 |
### 4.3 用户设备列表
```text
GET /internal/admin-query/tenants/{tenant_code}/users/{user_id}/devices
```
用途:用户详情中的登录设备列表。
建议查询参数:
| 参数 | 类型 | 说明 |
|---|---|---|
| `device_id` | string | 设备 ID。 |
| `platform` | string | 平台。 |
| `page` | int | 页码。 |
| `per_page` | int | 每页数量。 |
建议返回字段:
| 字段 | 说明 |
|---|---|
| `device_id` | 设备 ID。 |
| `platform` | 平台。 |
| `user_agent` | UA。 |
| `last_seen` | 最后在线。 |
| `created_at` | 创建时间。 |
| `updated_at` | 更新时间。 |
当前 Admin 直连涉及表:`im_devices`。
### 4.4 群列表
```text
GET /internal/admin-query/tenants/{tenant_code}/groups
```
用途:IM 群组管理列表。
建议查询参数:
| 参数 | 类型 | 说明 |
|---|---|---|
| `topic` | string | 群 Topic。 |
| `owner_id` | string | 群主 ID。 |
| `state` | int | Tinode topic 状态。 |
| `group_status` | string | 业务群状态。 |
| `created_at_start` | string | 创建时间开始。 |
| `created_at_end` | string | 创建时间结束。 |
| `touched_at_start` | string | 活跃时间开始。 |
| `touched_at_end` | string | 活跃时间结束。 |
| `page` | int | 页码。 |
| `per_page` | int | 每页数量。 |
建议返回字段:
| 字段 | 说明 |
|---|---|
| `topic` | 群 Topic。 |
| `state` | Tinode 状态。 |
| `owner_id` | 群主 ID。 |
| `seq_id` | 最新消息序号。 |
| `del_id` | 删除序号。 |
| `sub_count` | Tinode 订阅成员数。 |
| `touched_at` | 活跃时间。 |
| `created_at` | 创建时间。 |
| `updated_at` | 更新时间。 |
| `public` | Tinode public JSON。 |
| `trusted` | Tinode trusted JSON。 |
| `tags` | 标签。 |
| `aux` | 辅助 JSON。 |
| `group_name` | 群名称。 |
| `avatar_file_id` | 群头像文件 ID。 |
| `description` | 群简介。 |
| `group_status` | 业务群状态。 |
| `join_approval` | 入群审批开关。 |
| `mute_all` | 全员禁言。 |
| `member_upload_files` | 成员上传文件。 |
| `member_private_chat` | 成员私聊。 |
| `member_invite` | 成员邀请。 |
| `max_member_count` | 最大成员数。 |
| `max_admin_count` | 最大管理员数。 |
| `business_member_count` | 业务成员数。 |
| `settings_version` | 治理配置版本。 |
| `settings_updated_at` | 治理配置更新时间。 |
当前 Admin 直连涉及表:`im_topics`、`im_group_settings`。
### 4.5 群详情增强
现有接口:
```text
GET /internal/admin-query/tenants/{tenant_code}/groups/{topic}
```
当前接口主要返回 `im_group_settings`。建议增强 Tinode topic 基础字段:
| 字段 | 说明 |
|---|---|
| `state` | Topic 状态。 |
| `seq_id` | 最新消息序号。 |
| `del_id` | 删除序号。 |
| `sub_count` | 订阅数量。 |
| `public` | public JSON。 |
| `trusted` | trusted JSON。 |
| `access` | access JSON。 |
| `tags` | tags JSON。 |
| `aux` | aux JSON。 |
| `touched_at` | 活跃时间。 |
### 4.6 群成员列表增强
现有接口:
```text
GET /internal/admin-query/tenants/{tenant_code}/groups/{topic}/members
```
建议增强查询参数:
| 参数 | 类型 | 说明 |
|---|---|---|
| `user_id` | string | 用户 ID 精确过滤。 |
| `role` | string | `owner/admin/member`。 |
| `member_status` | string | 成员状态。 |
| `q` | string | 昵称、公开账号等关键词。 |
| `eligible_role` | string | 候选角色筛选。 |
| `page` | int | 页码。 |
| `per_page` | int | 每页数量。 |
| `cursor` | string | 兼容 cursor 分页。 |
| `limit` | int | 兼容 cursor 分页。 |
建议增强返回:
| 字段 | 说明 |
|---|---|
| `total` | 总数,Admin 表格需要。 |
| `items[].user_id` | 用户 ID。 |
| `items[].nickname` | 昵称。 |
| `items[].state` | 用户状态。 |
| `items[].mode_want` | 期望权限。 |
| `items[].mode_given` | 授予权限。 |
| `items[].recv_seq_id` | 接收序号。 |
| `items[].read_seq_id` | 已读序号。 |
| `items[].del_id` | 删除序号。 |
| `items[].role` | 业务角色。 |
| `items[].member_status` | 成员状态。 |
| `items[].group_nickname` | 群昵称。 |
| `items[].pinned` | 置顶。 |
| `items[].mute_notifications` | 免打扰。 |
| `items[].join_source` | 加入来源。 |
| `items[].joined_by` | 邀请或处理人。 |
| `items[].joined_at` | 加入时间。 |
| `items[].left_at` | 离开时间。 |
| `items[].removed_by` | 移除人。 |
| `items[].version` | 版本。 |
当前 Admin 直连涉及表:`im_group_members`、`im_subscriptions`、`im_users`、`im_user_profiles`。
### 4.7 入群申请列表
```text
GET /internal/admin-query/tenants/{tenant_code}/groups/{topic}/join-requests
```
建议查询参数:`status`、`applicant_user_id`、`page`、`per_page`。
建议返回字段:`request_id`、`topic`、`applicant_user_id`、`inviter_user_id`、`source`、`message`、`status`、`reject_reason`、`handled_by`、`handled_at`、`expires_at`、`version`、`created_at`、`updated_at`。
当前 Admin 直连涉及表:`im_group_join_requests`。
### 4.8 群邀请列表
```text
GET /internal/admin-query/tenants/{tenant_code}/groups/{topic}/invitations
```
建议查询参数:`status`、`target_user_id`、`inviter_user_id`、`page`、`per_page`。
建议返回字段:`invitation_id`、`topic`、`inviter_user_id`、`target_user_id`、`message`、`status`、`result`、`join_request_id`、`expires_at`、`handled_at`、`created_at`、`updated_at`。
当前 Admin 直连涉及表:`im_group_invitations`。
### 4.9 群公告列表
```text
GET /internal/admin-query/tenants/{tenant_code}/groups/{topic}/announcements
```
建议查询参数:`status`、`page`、`per_page`。
建议返回字段:`announcement_id`、`topic`、`author_user_id`、`content`、`status`、`version`、`published_at`、`deleted_by`、`deleted_at`、`created_at`、`updated_at`。
当前 Admin 直连涉及表:`im_group_announcements`。
### 4.10 群禁言列表
```text
GET /internal/admin-query/tenants/{tenant_code}/groups/{topic}/mutes
```
建议查询参数:`status`、`user_id`、`page`、`per_page`。
建议返回字段:`topic`、`user_id`、`muted_by`、`muted_at`、`expires_at`、`reason`、`status`、`revoked_by`、`revoked_at`、`version`、`created_at`、`updated_at`。
当前 Admin 直连涉及表:`im_group_mutes`。
### 4.11 群管理日志
```text
GET /internal/admin-query/tenants/{tenant_code}/groups/{topic}/admin-logs
```
建议查询参数:`action`、`operator_id`、`page`、`per_page`。
建议返回字段:`log_id`、`topic`、`operator_id`、`target_user_id`、`action`、`object_type`、`object_id`、`before_data`、`after_data`、`request_id`、`idempotency_key`、`created_at`。
当前 Admin 直连涉及表:`im_group_admin_logs`。
### 4.12 群文件列表
```text
GET /internal/admin-query/tenants/{tenant_code}/groups/{topic}/files
```
用途:群治理页面中的群文件列表。不要和已有单文件详情接口混淆。
建议查询参数:`status`、`file_type`、`file_name`、`uploader_id`、`page`、`per_page`。
建议返回字段:`group_file_id`、`topic`、`file_id`、`source_topic`、`source_seq`、`uploader_id`、`file_name`、`mime_type`、`file_type`、`file_size`、`status`、`deleted_by`、`deleted_at`、`created_at`、`updated_at`。
当前 Admin 直连涉及表:`im_group_files`。
### 4.13 文件列表
```text
GET /internal/admin-query/tenants/{tenant_code}/files
```
用途:IM 文件管理列表。
建议查询参数:`file_id`、`user_id`、`status`、`mime_type`、`created_at_start`、`created_at_end`、`page`、`per_page`。
建议返回字段:`id`、`file_id`、`user_id`、`mime_type`、`file_size`、`file_name`、`location`、`etag`、`status`、`quarantine_reason`、`created_at`、`updated_at`。
当前 Admin 直连涉及表:`im_fileuploads`。
### 4.14 文件引用列表
```text
GET /internal/admin-query/tenants/{tenant_code}/files/{file_id}/links
```
用途:文件详情中展示消息引用。
建议返回字段:`topic`、`seq_id`、`user_id`、`created_at`。
当前 Admin 直连涉及表:`im_filemsglinks`。
## 5. P1:后台审核和运营页面
### 5.1 审计日志
```text
GET /internal/admin-query/tenants/{tenant_code}/audit-logs
GET /internal/admin-query/tenants/{tenant_code}/audit-logs/{id}
```
建议查询参数:`actor_user_id`、`action`、`target_type`、`target_id`、`created_at_start`、`created_at_end`、`page`、`per_page`。
建议返回字段:`id`、`request_id`、`actor_user_id`、`action`、`target_type`、`target_id`、`ip`、`user_agent`、`detail`、`created_at`。
当前 Admin 直连涉及表:`im_audit_logs`。
### 5.2 实名审核
```text
GET /internal/admin-query/tenants/{tenant_code}/realname-verifications
GET /internal/admin-query/tenants/{tenant_code}/realname-verifications/{id}
```
建议查询参数:`user_id`、`status`、`real_name`、`created_at_start`、`created_at_end`、`page`、`per_page`。
建议返回字段:`id`、`user_id`、`real_name`、`id_card_no_masked`、`front_file_id`、`back_file_id`、`status`、`reject_reason`、`reviewed_by`、`reviewed_at`、`created_at`、`updated_at`。
当前 Admin 直连涉及表:`im_realname_verifications`。
### 5.3 举报工单
```text
GET /internal/admin-query/tenants/{tenant_code}/abuse-reports
GET /internal/admin-query/tenants/{tenant_code}/abuse-reports/{id}
GET /internal/admin-query/tenants/{tenant_code}/reports/abuse/{report_id}/target
```
建议查询参数:`reporter_user_id`、`target_type`、`target_id`、`status`、`created_at_start`、`created_at_end`、`page`、`per_page`。
建议返回字段:`id`、`report_id`、`reporter_user_id`、`target_type`、`target_id`、`reason_code`、`description`、`evidence_file_ids`、`status`、`handler_id`、`handled_at`、`handle_result`、`created_at`、`updated_at`。
当前 Admin 直连涉及表:`im_abuse_reports`。
### 5.4 意见反馈
```text
GET /internal/admin-query/tenants/{tenant_code}/feedback-submissions
GET /internal/admin-query/tenants/{tenant_code}/feedback-submissions/{id}
```
建议查询参数:`user_id`、`category`、`status`、`created_at_start`、`created_at_end`、`page`、`per_page`。
建议返回字段:`id`、`feedback_id`、`user_id`、`category`、`content`、`contact`、`file_ids`、`status`、`handler_id`、`handled_at`、`reply`、`created_at`、`updated_at`。
当前 Admin 直连涉及表:`im_feedback_submissions`。
### 5.5 业务 Outbox
```text
GET /internal/admin-query/tenants/{tenant_code}/outbox/business-events
GET /internal/admin-query/tenants/{tenant_code}/outbox/group-events
GET /internal/admin-query/tenants/{tenant_code}/outbox/business-events/{id}
GET /internal/admin-query/tenants/{tenant_code}/outbox/group-events/{id}
```
建议查询参数:`event_type`、`status`、`target_type`、`target`、`created_at_start`、`created_at_end`、`page`、`per_page`。
当前 Admin 直连涉及表:`im_business_events_outbox`、`im_group_events_outbox`。
## 6. P2:如果要完全消除租户库直连
这些不是 Kafka 联调第一优先级,但如果最终要求 Admin 完全不碰租户库,也需要 IM Server 提供内部查询或改为 hotgo 摘要表。
### 6.1 组织架构
```text
GET /internal/admin-query/tenants/{tenant_code}/org/departments
GET /internal/admin-query/tenants/{tenant_code}/org/members
```
当前 Admin 直连涉及表:`im_org_departments`、`im_org_members`、`im_users`。
### 6.2 工作台应用
```text
GET /internal/admin-query/tenants/{tenant_code}/workbench/apps
```
当前 Admin 直连涉及工作台相关租户表。
### 6.3 租户任务看板摘要
```text
GET /internal/admin-query/tenants/{tenant_code}/dashboard/summary
```
当前 Admin 会统计多个租户库中的待处理实名、举报、反馈、Outbox 等数量。
### 6.4 租户配置类查询
如果这些配置仍放在租户库,也需要补接口;如果后续迁到 hotgo 控制库,则不需要通过 IM Server 查询。
```text
GET /internal/admin-query/tenants/{tenant_code}/feedback-config
GET /internal/admin-query/tenants/{tenant_code}/mailbox-config
GET /internal/admin-query/tenants/{tenant_code}/abuse-reasons
GET /internal/admin-query/tenants/{tenant_code}/app-releases
```
涉及模块:反馈配置、总经理信箱配置、举报原因、App 发布策略。
## 6.5 IM Server 补齐状态
截至 2026-08-27,IM Server 已补齐本清单中的内部查询入口,统一走 `/internal/admin-query/tenants/{tenant_code}/...`,并沿用 HMAC 鉴权。
已接入范围:
1. P0:用户列表、用户详情增强、用户设备、群列表、群详情增强、群成员 total、入群申请、群邀请、群公告、群禁言、群管理日志、群文件、文件列表、文件详情、文件引用。
2. P1:审计日志、实名审核、举报工单、举报目标快照、意见反馈、业务 Outbox、群 Outbox。
3. P2:组织部门、组织成员、工作台应用、任务看板摘要、反馈配置、总经理信箱配置、举报原因、App 发布策略。
实现说明:
1. IM Server 内部使用白名单只读查询规格,不提供任意 SQL 入口。
2. 列表接口统一返回 `items/total/page/per_page`。
3. 详情接口统一返回单对象。
4. 复杂字段先返回租户库原始 JSON 或轻量关联摘要,Admin 联调时如需展示专用字段再做二次增强。
5. 若租户库未创建某些 P1/P2 表,对应接口会返回内部错误;发版前需要确保租户库 schema 已完整迁移。
## 7. Admin 侧迁移建议
### 7.1 第一批迁移
在 IM Server 补完 P0 后,Admin 优先迁移:
1. 用户列表、用户详情、用户设备。
2. 群列表、群详情、群成员。
3. 群入群申请、邀请、公告、禁言、管理日志、群文件。
4. 文件列表、文件详情、文件引用。
### 7.2 第二批迁移
在 IM Server 补完 P1 后,Admin 再迁移:
1. 审计日志。
2. 实名审核。
3. 举报工单。
4. 意见反馈。
5. 业务 Outbox。
### 7.3 保留 hotgo 查询的范围
以下数据天然属于 hotgo 控制库,Admin 可继续查 hotgo,不需要走 IM Server:
1. 租户表和租户绑定表。
2. Admin 命令表:`im_admin_commands`。
3. Admin Outbox 表:`im_admin_outbox`。
4. Admin Kafka 审计事件表:`im_audit_events`。
5. 菜单、角色、管理员、系统配置、字典等 HotGo 系统表。
## 8. 验收标准
1. Admin 查询租户业务数据时,不再出现新的 `imtenant.DB()` 调用。
2. Admin 查询 IM Server 内部接口时,统一使用 HMAC 签名。
3. 列表接口支持 `page/per_page/total`,兼容后台表格分页。
4. 详情接口返回结构化 JSON,不返回数据库内部敏感字段。
5. 用户 ID、雪花 ID 等大整数以字符串返回。
6. IM Server 为内部查询写审计日志,至少包含 `tenant_code`、`service_name`、`path`、`query`、`trace_id`、`status`、`duration_ms`。
7. 关闭 IM Server 内部查询能力时,Admin 应得到明确错误,不泄露租户库信息。
+953
View File
@@ -0,0 +1,953 @@
# IM Kafka 接入方案
> 日期:2026-08-26
> 关联文档:[IM Kafka 接入范围识别](./IM-Kafka接入范围识别.md)、[IM 前后端逐图对接协议](./IM-前后端对接文档.md)
> 目标:明确 Admin 后台、IM 服务、HotGo 控制库、租户业务库之间的 Kafka 接入边界、查询方式、鉴权方式和首批落地范围。
## 0. 与范围识别文档的关系
[IM Kafka 接入范围识别](./IM-Kafka接入范围识别.md) 是本方案的前置输入,负责按前后端对接文档逐章识别哪些业务适合 Kafka、哪些仍应走 WebSocket/Redis/MySQL/OSS。本方案在该识别结果基础上,进一步确定服务边界、库访问边界、前后台交互模型、内部查询接口、鉴权方式和首批落地步骤。
两份文档分工如下:
| 文档 | 作用 |
| --- | --- |
| [IM Kafka 接入范围识别](./IM-Kafka接入范围识别.md) | 回答“哪些业务该用 Kafka,哪些不该用”。 |
| 本方案 | 回答“Kafka 怎样接入,Admin 与 IM 怎样交互,服务和库怎样隔离”。 |
实施时以范围识别文档的逐章清单确定业务覆盖面,以本方案确定技术实现方式。若后续新增需求,先补充范围识别,再同步调整本方案的 Topic、Command、内部接口和鉴权规则。
## 1. 核心原则
系统按“服务和库一一对应”设计,不允许不同服务直连对方数据库。
| 服务 | 只能直接访问 | 说明 |
| --- | --- | --- |
| Admin/HotGo 服务 | `hotgo` 控制库 | 管理后台、租户主数据、后台配置、审核、操作单。 |
| IM 服务 | 当前租户业务库 | 用户、好友、群、消息、文件引用、工作台业务数据。 |
| Redis | IM 实时状态与缓存 | 在线状态、短状态、热点消息、缓存分层。 |
| Kafka | Admin 与 IM 的命令、事件、结果桥 | 后台写操作、异步任务、业务事件、审计、缓存失效。 |
| OSS | 文件对象存储 | 上传下载字节流不进入 Kafka。 |
最终边界:
1. Admin 不直连租户库。
2. IM 不直连 `hotgo` 库。
3. Admin 写租户业务操作统一通过 Kafka Command。
4. IM 写后台业务状态不直写 `hotgo`,通过 Kafka Result/Event 返回 Admin。
5. Admin 查询租户业务数据通过 IM 内部只读 API。
6. IM 查询租户配置、字典、路由通过 Admin 内部只读 API,或消费 Kafka 配置事件维护本地副本。
7. 在线实时投递继续走 Redis + WebSocket,Kafka 不替代 WS 主链路。
## 2. 总体架构
```text
Admin/HotGo 服务
-> 写 hotgo 操作单
-> 生产 Kafka Command
-> IM 服务消费 Command
-> 写租户业务库
-> 生产 Kafka Result/Event
-> Admin 消费 Result/Event
-> 更新 hotgo 操作状态
Admin 查询租户业务数据
-> 调 IM 内部只读 API
-> IM 查租户业务库
-> 返回 Admin
IM 查询租户配置/字典/路由
-> 优先读本地缓存/本地副本
-> 缓存缺失时调 Admin 内部只读配置 API
-> Admin 查 hotgo
-> 返回 IM
```
## 3. Kafka 使用范围
| 场景 | 是否使用 Kafka | 说明 |
| --- | --- | --- |
| Admin 后台群操作通知 C 端 | 是 | 后台写 hotgo 操作单,投 Kafka,IM 执行租户库变更并 WS 下发。 |
| Admin 后台用户封禁、解封、重置密码 | 是 | IM 消费后更新租户库、踢在线、通知客户端。 |
| Admin 后台租户配置、字典、功能开关变更 | 是 | Admin 写 hotgo 后投配置事件,IM 刷新缓存或本地副本。 |
| 企业注册、实名、举报、反馈审核结果 | 是 | Admin 审核后投 Kafka,IM 生成通知或更新租户侧状态。 |
| 工作台会议、日报、月报、信箱通知 | 是 | 状态变更、提醒、红点更新走 Kafka。 |
| 私聊/群聊消息异步落库 | 是 | WS 收消息后完成鉴权、幂等和 seq,再投 Kafka 异步持久化。 |
| 文件安全检测、恶意文件检测 | 是 | OSS 上传完成后投检测任务,消费者调用第三方并更新状态。 |
| 离线 Push | 是 | 消息或业务事件落地后投 Push 任务。 |
| 在线消息实时 fanout | 不作为主链路 | 继续 Redis Pub/Sub/Stream + WS。 |
| 文件上传下载字节流 | 否 | 走 OSS/文件服务;Kafka 只处理上传完成后的事件。 |
| 验证码、Action Token、限流、在线状态 | 否 | 继续 Redis。 |
| 普通查询接口 | 否 | 走内部只读 API 或本服务数据库。 |
## 4. 前后台交互模型
### 4.1 后台写租户业务
Admin 后台所有会影响租户业务库或 C 端展示的写操作,统一走 Kafka Command。
```text
Admin
1. 校验后台管理员权限
2. 写 hotgo.admin_command,状态 pending
3. 发送 Kafka command
IM
4. 消费 command
5. 校验租户、对象、版本、权限和幂等
6. 写租户业务库
7. 必要时通过 WS 通知在线 C 端
8. 发送 Kafka result
Admin
9. 消费 result
10. 更新 hotgo.admin_command 状态
```
典型操作:
| 操作 | Command | IM 执行动作 | C 端通知 |
| --- | --- | --- | --- |
| 群资料修改 | `group.profile.update` | 更新群表和 Topic 资料 | `group.profile.updated` |
| 设置/删除管理员 | `group.admin.set` / `group.admin.remove` | 更新成员角色 | `group.role.updated` |
| 禁言/解禁 | `group.member.mute` / `group.member.unmute` | 更新禁言表 | `group.mute.updated` |
| 踢人 | `group.member.remove` | 更新成员状态和订阅 | `group.member.removed` |
| 解散群 | `group.delete` | 解散 Topic、撤销权限 | `group.deleted` |
| 用户封禁 | `user.disable` | 更新用户状态、踢在线 | `user.disabled` |
| 重置密码 | `user.password.reset` | 更新密码和 `auth_version` | 可选安全通知 |
### 4.2 后台查询租户业务
Kafka 不适合做普通查询。Admin 需要实时查看租户业务数据时,调用 IM 提供的内部只读 API。
只读 API 只允许 `GET`,不提供通用 SQL 或万能查询接口。
| 接口 | 用途 |
| --- | --- |
| `GET /internal/admin-query/tenants/{tenant_code}/users/{user_id}` | 查询租户用户详情。 |
| `GET /internal/admin-query/tenants/{tenant_code}/groups/{topic}` | 查询群详情。 |
| `GET /internal/admin-query/tenants/{tenant_code}/groups/{topic}/members` | 查询群成员。 |
| `GET /internal/admin-query/tenants/{tenant_code}/messages/{topic}/{seq}` | 查询消息受控快照。 |
| `GET /internal/admin-query/tenants/{tenant_code}/files/{file_id}` | 查询文件引用和审核状态。 |
| `GET /internal/admin-query/tenants/{tenant_code}/reports/abuse/{report_id}/target` | 查询举报目标快照。 |
| `GET /internal/admin-query/tenants/{tenant_code}/kafka/dead-letters` | 查询 IM 租户库 Kafka 死信列表。 |
| `GET /internal/admin-query/tenants/{tenant_code}/kafka/dead-letters/{dead_id}` | 查询单条死信详情和原始 envelope。 |
后台列表、统计、高频页面优先使用 `hotgo` 摘要表。摘要由 IM 通过 Kafka 事件同步给 Admin,Admin 查询自己的库。
推荐摘要表:
| 摘要表 | 来源 |
| --- | --- |
| `admin_im_user_snapshot` | IM 用户资料变更事件。 |
| `admin_im_group_snapshot` | IM 群资料、成员数、状态变更事件。 |
| `admin_im_group_member_stat` | 群成员统计事件。 |
| `admin_im_file_audit_snapshot` | 文件审核和引用状态事件。 |
| `admin_im_workbench_snapshot` | 工作台红点、会议、日报/月报状态事件。 |
### 4.3 IM 查询后台配置
IM 不直连 `hotgo`。IM 需要租户路由、租户状态、字典、第三方配置时,使用两种方式:
1. 常态:Admin 修改配置后发送 Kafka 配置事件,IM 消费后更新进程内缓存或本地副本。
2. 兜底:IM 缓存缺失或启动预热时,调用 Admin 内部只读配置 API。
| 接口 | 用途 |
| --- | --- |
| `GET /internal/im-config/tenants/{tenant_code}` | 查询租户状态、名称、路由标识。 |
| `GET /internal/im-config/tenants/{tenant_code}/limits` | 查询人数、群数、上传大小等限制。 |
| `GET /internal/im-config/tenants/{tenant_code}/features` | 查询功能开关。 |
| `GET /internal/im-config/dictionaries?types=oss,file_security,sms,jpush,agora` | 查询字典配置。 |
| `GET /internal/im-config/app/releases/latest` | 查询 App 版本配置。 |
Admin 内部配置 API 只返回 IM 运行需要的字段,不返回后台管理员账号、菜单权限、审核私有字段。
## 5. Topic 规划
| Topic | 用途 |
| --- | --- |
| `im.admin.command` | Admin 发给 IM 的租户业务写命令。 |
| `im.admin.command.result` | IM 回传命令执行结果。 |
| `im.config.event` | Admin 发给 IM 的配置、字典、租户状态变更。 |
| `im.business.event` | IM 内部或 Admin 触发的 C 端业务事件。 |
| `im.message.persist` | 私聊、群聊消息异步落库。 |
| `im.message.push` | 离线 Push 任务。 |
| `im.file.security` | 文件内容安全、恶意文件检测任务。 |
| `im.file.lifecycle` | 文件引用绑定、释放、回收、合规删除。 |
| `im.workbench.event` | 会议、日报、月报、信箱、工作台红点事件。 |
| `im.cache.invalidate` | 跨节点缓存失效。 |
| `im.audit.event` | 审计事件。 |
| `im.dead.letter` | 消费失败死信。 |
首版可合并部分 Topic,但事件体必须保留 `biz_type` 和 `event_type`,方便后续拆分。
## 6. 消息结构
统一 Kafka Envelope:
```json
{
"event_id": "evt_01K...",
"trace_id": "req_01K...",
"tenant_code": "default",
"source_service": "hotgo-admin",
"target_service": "im",
"event_type": "group.member.mute",
"biz_type": "group",
"biz_id": "grp...",
"idempotency_key": "idem_01K...",
"version": 3,
"occurred_at": "2026-08-26T10:00:00.000Z",
"actor": {
"type": "admin",
"id": "admin_1",
"display_name": "管理员"
},
"payload": {}
}
```
Command Result:
```json
{
"event_id": "evt_result_01K...",
"trace_id": "req_01K...",
"tenant_code": "default",
"command_id": "cmd_01K...",
"command_event_id": "evt_01K...",
"status": "succeeded",
"reason": "",
"message": "",
"result": {},
"occurred_at": "2026-08-26T10:00:01.000Z"
}
```
失败时 `status=failed`,`reason` 使用稳定错误码,例如 `GROUP_VERSION_CONFLICT`、`FORBIDDEN`、`RESOURCE_NOT_FOUND`、`COMMAND_DUPLICATED`。
## 7. 幂等、事务与重试
### 7.1 生产端
Admin 写操作必须先落 `hotgo.admin_command`,再发送 Kafka。推荐使用 outbox 模式:
1. 同一事务写 `admin_command` 和 `admin_outbox`。
2. outbox poller 发送 Kafka。
3. Kafka 发送成功后标记 outbox 已发送。
4. 发送失败按退避重试。
IM 内部业务事件也按同样方式写租户库 outbox,避免“数据库成功、事件丢失”。
### 7.2 消费端
每个消费者维护消费幂等记录:
| 字段 | 说明 |
| --- | --- |
| `event_id` | Kafka 事件唯一 ID。 |
| `consumer_group` | 消费组。 |
| `status` | `processing/succeeded/failed/dead`。 |
| `retry_count` | 重试次数。 |
| `last_error` | 最近失败原因。 |
| `created_at/updated_at` | 时间。 |
消费前先插入或锁定 `event_id + consumer_group`。已成功的事件直接跳过;处理中超时可重新抢占。业务表必须再用唯一键兜底,例如 `tenant + command_id`、`tenant + client_mid`、`tenant + topic + seq`。
### 7.3 重试与死信
| 类型 | 策略 |
| --- | --- |
| 临时失败 | 指数退避重试。 |
| 版本冲突 | 不自动重试,回传 failed,由 Admin 刷新后重新发命令。 |
| 权限失败 | 不重试。 |
| 对象不存在 | 不重试,除非明确是延迟同步。 |
| 第三方失败 | 可重试,超过次数写死信和失败表。 |
### 7.4 消费失败处理流程
消费者收到消息后按固定流程处理:
```text
收到 Kafka 消息
-> 校验 Envelope 格式、签名字段、租户和事件类型
-> 抢占 event_id + consumer_group 幂等记录
-> 已 succeeded:提交 offset,跳过
-> processing 超时:允许重新抢占
-> 执行业务逻辑
-> 成功:标记 succeeded,提交 offset
-> 可重试失败:retry_count + 1,按退避策略延迟重试
-> 超过最大重试:写入死信,标记 dead,提交 offset
-> 不可重试失败:标记 failed 或 dead,必要时回传 command result,提交 offset
```
重试建议参数:
| 参数 | 建议值 | 说明 |
| --- | --- | --- |
| `max_retry_count` | 5~10 | 第三方接口可取高值,权限/参数错误不重试。 |
| `retry_backoff` | 10s、30s、1m、5m、15m | 指数退避,可加随机抖动。 |
| `processing_timeout` | 5m~30m | 超过后允许其他消费者重新抢占。 |
| `dead_letter_retention` | 30~90 天 | 便于排障和人工补偿。 |
### 7.5 死信队列处理
进入死信表示该事件已超过自动恢复能力,系统不能再无限重试。死信必须形成“可查看、可诊断、可重放、可关闭”的闭环。
死信写入两处:
1. Kafka `im.dead.letter` Topic:保留原始事件,便于流式订阅、告警和后续自动补偿。
2. 数据库失败表:便于 Admin 后台查询、人工处理和审计。
推荐失败表字段:
| 字段 | 说明 |
| --- | --- |
| `dead_id` | 死信记录 ID。 |
| `event_id` | 原 Kafka 事件 ID。 |
| `source_topic` | 原 Topic。 |
| `consumer_group` | 失败消费组。 |
| `tenant_code` | 租户编码。 |
| `event_type` | 事件类型。 |
| `biz_type` / `biz_id` | 业务对象。 |
| `payload` | 原始 Envelope。 |
| `retry_count` | 已重试次数。 |
| `last_reason` | 稳定失败原因。 |
| `last_error` | 最近错误摘要,敏感字段脱敏。 |
| `status` | `dead/pending_replay/replayed/ignored/resolved`。 |
| `replay_count` | 人工重放次数。 |
| `created_at/updated_at` | 时间。 |
| `resolved_by/resolved_at` | 人工处理人和处理时间。 |
| `resolve_note` | 处理备注。 |
死信来源分类:
| 分类 | 示例 | 默认处理 |
| --- | --- | --- |
| 临时资源失败超过上限 | DB 短暂不可用、第三方超时 | 修复环境后允许重放。 |
| 业务状态冲突 | 群版本冲突、对象已被删除 | 不自动重放,Admin 刷新状态后重新发命令。 |
| 权限失败 | 管理员无权限、租户停用 | 不重放,标记 ignored 或 failed result。 |
| 参数错误 | Envelope 缺字段、payload 非法 | 不重放,修复生产方后关闭。 |
| 第三方永久失败 | 文件违规、Push token 无效 | 按业务结果落状态,不重放或仅重放特定步骤。 |
Admin 后台需要提供死信处理页:
| 操作 | 说明 |
| --- | --- |
| 查看详情 | 展示事件、租户、业务对象、错误原因、重试次数和 trace_id。 |
| 重放 | 将原事件或修正后的事件重新投递到原 Topic,生成新的 `event_id`,保留 `parent_event_id`。 |
| 忽略 | 确认该事件无需继续处理,状态改为 `ignored`。 |
| 标记已处理 | 线下补偿完成后改为 `resolved`。 |
| 重新生成命令 | 对版本冲突类命令,不直接重放旧事件,而是基于最新状态创建新 Command。 |
重放规则:
1. 重放必须生成新的 `event_id`,原 `event_id` 保留在 `parent_event_id`。
2. 重放前重新校验租户状态、对象是否存在、操作者权限和业务版本。
3. 参数错误、权限失败、版本冲突默认禁止直接重放。
4. 文件安全检测、Push、审计写入等可重试任务允许重放。
5. 每次重放都写审计,记录处理人、原因、原事件和新事件。
死信告警:
| 指标 | 建议 |
| --- | --- |
| 单 Topic 死信数 | 5 分钟内超过阈值告警。 |
| 同一 `event_type` 连续失败 | 可能是代码或配置问题,立即告警。 |
| 同一租户大量失败 | 检查租户库、配置或权限状态。 |
| 死信积压时间 | 超过 24 小时未处理提醒。 |
对 Admin Command 类事件,进入死信后必须回传 `im.admin.command.result`,状态为 `failed` 或 `dead`,让后台操作单结束等待状态。否则后台会长期显示处理中。
## 8. 内部接口鉴权
内部 API 鉴权非常关键,必须同时做网络层、服务层和业务层控制。
### 8.1 网络层
1. `/internal/*` 只允许内网网关或内网地址访问。
2. 禁止公网直接暴露内部接口。
3. 有条件时启用 mTLS。
### 8.2 服务签名
内部请求统一携带:
| Header | 说明 |
| --- | --- |
| `X-Service-Name` | 调用方服务名,例如 `hotgo-admin` 或 `im-server`。 |
| `X-Timestamp` | Unix 秒或毫秒时间戳。 |
| `X-Nonce` | 随机串,短期内不可重复。 |
| `X-Request-ID` | 请求追踪 ID。 |
| `X-Signature` | HMAC 签名。 |
签名内容:
```text
METHOD + "\n" +
PATH_WITH_QUERY + "\n" +
BODY_SHA256 + "\n" +
X-Timestamp + "\n" +
X-Nonce
```
校验规则:
1. 时间窗建议 5 分钟。
2. `nonce` 写 Redis,TTL 5 分钟,防重放。
3. 每个服务独立 `service_id/service_secret`。
4. 每个服务配置允许访问的内部接口白名单。
### 8.3 业务授权
Admin 调 IM 查询时,签名 payload 或 Header 中必须包含后台操作者信息:
| 字段 | 说明 |
| --- | --- |
| `admin_user_id` | 后台用户 ID。 |
| `admin_role` | 后台角色。 |
| `admin_permissions` | 本次已校验过的权限列表。 |
| `tenant_code` | 被查询租户。 |
| `reason` | 查询原因或工单号。 |
IM 仍需根据 `tenant_code + permission + resource` 做二次校验,并写内部查询审计。
IM 调 Admin 配置接口时,Admin 只校验服务身份和接口白名单,不返回超出 IM 运行需要的字段。
## 9. 文件与工作台
### 9.1 文件
文件上传下载不走 Kafka,OSS 是文件字节流主链路。Kafka 只处理上传完成后的事件:
1. `im.file.security`:内容安全、恶意文件检测。
2. `im.file.lifecycle`:业务引用绑定、释放、临时文件回收、合规删除。
3. `im.business.event`:文件审核失败、合规处置等 C 端通知。
### 9.2 工作台
工作台查询不走 Kafka,状态变化和提醒走 Kafka:
1. 会议创建、更新、取消、提醒、响应:`im.workbench.event`。
2. 日报/月报提交、撤回、重开、催办:`im.workbench.event`。
3. 总经理信箱提交、处理结果:`im.workbench.event` + `im.audit.event`。
4. 工作台模块配置变更:`im.config.event` 或 `im.cache.invalidate`。
## 10. 首批落地建议
| 阶段 | 内容 |
| --- | --- |
| 第一步 | 增加 Kafka 配置、生产者、消费者、健康检查、统一 Envelope。 |
| 第二步 | 增加消费幂等表、outbox 表、死信表或死信 Topic。 |
| 第三步 | 接入 `im.admin.command` 和 `im.admin.command.result`,先覆盖后台群治理操作。 |
| 第四步 | 增加 IM 内部只读查询 API,供 Admin 查用户、群、消息、文件快照。 |
| 第五步 | 增加 Admin 内部配置 API,供 IM 查询租户配置、字典、第三方参数。 |
| 第六步 | 接入文件安全检测、离线 Push、工作台事件。 |
| 第七步 | 消息异步落库接入 `im.message.persist`,完善幂等和重试。 |
## 11. 分端任务拆分
### 11.1 Admin/HotGo 侧任务
Admin 侧只访问 `hotgo` 库,负责后台操作入口、配置主数据、命令生产、结果消费和死信处理。
| 模块 | 任务 | 产出 |
| --- | --- | --- |
| Kafka 基础 | 增加 Kafka producer/consumer 配置、健康检查、统一 Envelope 组装。 | 可发送 `im.admin.command`、消费 `im.admin.command.result`。 |
| 操作单 | 新增 `admin_command`、`admin_outbox` 表和状态机。 | `pending/running/succeeded/failed/dead` 操作状态。 |
| Outbox | 后台写操作单与 outbox 同事务落库,poller 发送 Kafka。 | 防止 hotgo 写成功但 Kafka 丢事件。 |
| Command 生产 | 群治理、用户封禁/解封、重置密码、审核结果、工作台配置等后台操作生成 Command。 | `im.admin.command` 事件。 |
| Result 消费 | 消费 IM 回传结果并更新操作单。 | 后台页面可展示最终成功/失败原因。 |
| 配置事件 | 租户配置、字典、功能开关、第三方配置变更后发送 `im.config.event`。 | IM 刷新缓存或本地副本。 |
| 配置查询 API | 提供 `/internal/im-config/*` 只读接口。 | IM 启动预热和缓存缺失兜底查询。 |
| 摘要表 | 建立用户、群、文件、工作台等后台摘要表。 | 后台列表统计优先查 hotgo。 |
| 死信后台 | 提供死信查看、重放、忽略、标记已处理、重新生成命令。 | 运维闭环。 |
| 内部鉴权 | 实现服务签名校验、nonce 防重放、接口白名单、审计。 | 保护 Admin 内部配置 API。 |
| 审计 | 记录后台操作者、租户、资源、Command、Result、死信处理。 | 可追踪后台操作链路。 |
Admin 首批优先级:
1. Kafka 基础配置、Envelope、producer。
2. `admin_command` + `admin_outbox`。
3. 群治理 Command 生产。
4. `im.admin.command.result` 消费。
5. `/internal/im-config/*` 配置查询 API。
6. 死信管理页。
### 11.2 IM Server 侧任务
IM Server 侧只访问租户业务库,负责执行 Admin Command、维护租户业务状态、通过 WS 通知 C 端,并向 Admin 回传结果。
| 模块 | 任务 | 产出 |
| --- | --- | --- |
| Kafka 基础 | 增加 Kafka producer/consumer 配置、健康检查、统一 Envelope 解析。 | 可消费 Command、生产 Result/Event。 |
| 消费幂等 | 新增租户库消费幂等表或统一消费日志。 | `event_id + consumer_group` 幂等。 |
| Command 消费 | 消费 `im.admin.command`。 | 后台命令进入 IM 执行链路。 |
| 群治理执行器 | 执行群资料、管理员、禁言、踢人、解散、群主转让等命令。 | 更新租户库和 Topic 状态。 |
| 用户治理执行器 | 执行封禁、解封、重置密码、踢在线等命令。 | 更新用户状态和 `auth_version`。 |
| Result 生产 | 执行成功/失败后发送 `im.admin.command.result`。 | Admin 操作单闭环。 |
| WS 通知 | Command 成功后生产 C 端业务事件并投递在线客户端。 | `info.what="im|group"` 下发。 |
| 配置消费 | 消费 `im.config.event`,刷新进程内缓存或本地副本。 | 租户配置、字典、第三方配置生效。 |
| 配置兜底查询 | 调 Admin `/internal/im-config/*`。 | IM 启动预热和缓存缺失恢复。 |
| 内部只读 API | 提供 `/internal/admin-query/*`。 | Admin 实时查询租户业务数据。 |
| 文件任务 | 上传完成后生产/消费 `im.file.security`、`im.file.lifecycle`。 | 文件审核、引用绑定、回收。 |
| 消息异步落库 | 接入 `im.message.persist`。 | 私聊/群聊消息削峰落库。 |
| Push 任务 | 生产 `im.message.push`。 | 离线 Push 异步投递。 |
| 死信处理 | 写失败表、投 `im.dead.letter`、Command 死信回传 Result。 | 失败可诊断、可补偿。 |
| 内部鉴权 | 校验 Admin 调用 `/internal/admin-query/*` 的签名、权限和审计。 | 保护租户业务查询。 |
IM Server 首批优先级:
1. Kafka 基础配置、Envelope、consumer/producer。
2. 消费幂等表和死信写入。
3. `im.admin.command` 群治理消费执行。
4. `im.admin.command.result` 回传。
5. WS 业务事件下发。
6. `/internal/admin-query/*` 只读查询 API。
### 11.3 双方联调契约
| 契约 | Admin 负责 | IM Server 负责 |
| --- | --- | --- |
| Envelope 字段 | 生产完整字段,保证 `event_id/trace_id/tenant_code/event_type/idempotency_key`。 | 校验字段完整性和事件类型白名单。 |
| Command 状态 | 创建操作单,展示状态。 | 回传成功、失败、死信结果。 |
| 幂等 | 同一后台操作复用 `command_id/idempotency_key`。 | 同一 Command 重复消费只执行一次。 |
| 错误码 | 展示 IM 返回的稳定 `reason`。 | 返回稳定 `reason`,不让 Admin 解析错误文本。 |
| 查询接口 | 带管理员身份、权限、原因和签名。 | 校验签名、权限和租户资源,写审计。 |
| 配置接口 | 提供最小必要配置字段。 | 不直连 hotgo,使用缓存、事件或内部 API。 |
| 死信 | 提供处理入口和人工操作审计。 | 写死信、回传 Command dead/failed 状态。 |
| Trace | 生成并传递 `trace_id`。 | 日志、Kafka、WS 下发沿用同一 `trace_id`。 |
### 11.4 首批联调用例
| 用例 | 预期 |
| --- | --- |
| Admin 禁言群成员 | Admin 操作单 pending,IM 消费后更新租户库,C 端收到 `group.mute.updated`,Admin 状态 succeeded。 |
| Admin 重复提交同一禁言操作 | IM 只执行一次,Result 返回同一最终状态。 |
| Admin 删除管理员但群版本冲突 | IM 返回 `GROUP_VERSION_CONFLICT`,Admin 操作单 failed,不自动重试。 |
| IM 消费时临时 DB 失败 | 自动退避重试,未超限不进入死信。 |
| IM 消费超过重试上限 | 写 `im.dead.letter`,Admin 操作单进入 dead/failed。 |
| Admin 查询群成员 | Admin 调 `/internal/admin-query/*`,IM 校验签名和权限后返回只读数据。 |
| IM 启动加载租户配置 | IM 优先读缓存,缺失时调 `/internal/im-config/*`,Admin 只返回必要配置。 |
### 11.5 IM Server 已接入的首批 Admin Command
IM Server 首批消费 `im.admin.command`,执行后回传 `im.admin.command.result`。所有命令必须带 `tenant_code`,`payload.actor_user_id` 必须是租户内 IM 用户 ID,用于落群事件和 C 端通知。
| Command `event_type` | 说明 | 必填 payload |
| --- | --- | --- |
| `group.profile.update` | 修改群名称、头像、简介。 | `command_id,actor_user_id,topic,profile,version` |
| `group.governance.update` | 修改进群审批、全员禁言、成员邀请等治理开关。 | `command_id,actor_user_id,topic,governance,version` |
| `group.admin.set` | 设置一个或多个管理员。 | `command_id,actor_user_id,topic,user_ids` |
| `group.admin.remove` | 删除管理员,降为普通成员。 | `command_id,actor_user_id,topic,user_id` |
| `group.member.mute` | 禁言成员。 | `command_id,actor_user_id,topic,user_id,reason,expires_at` |
| `group.member.unmute` | 解除禁言。 | `command_id,actor_user_id,topic,user_id` |
| `group.member.remove` | 移出群成员。 | `command_id,actor_user_id,topic,user_id` |
| `group.owner.transfer` | 转让群主。 | `command_id,actor_user_id,topic,target_user_id,version` |
当前 IM Server 消费 `im.admin.command` 时已经接入 `im_kafka_consume_log`:
1. 按 `event_id + consumer_group` 抢占消费。
2. 已成功事件直接跳过,不重复执行业务。
3. 处理中且未超时的事件跳过,避免并发重复执行。
4. 失败或处理超时的事件允许重新抢占。
5. 不可重试错误或超过重试次数后写入 `im_kafka_dead_letters`,并投递 `im.dead.letter`。
6. Admin Command 进入死信时回传 `im.admin.command.result`,`status=dead`。
### 11.6 IM Server 已接入的配置事件
IM Server 首批消费 `im.config.event`,用于 Admin 修改 HotGo 配置后通知 IM 刷新进程内缓存。当前实现以“缓存失效”为主,收到事件后清除对应缓存,下次业务读取时重新加载。
| Config `event_type` | 说明 | IM 行为 |
| --- | --- | --- |
| `tenant.updated` | 租户基础信息变更。 | 清理租户 code/id 缓存。 |
| `tenant.status.changed` | 租户启停、状态变化。 | 清理租户 code/id 缓存。 |
| `tenant.limits.updated` | 租户人数、群数、上传限制等变化。 | 清理租户 code/id 缓存。 |
| `tenant.features.updated` | 租户功能开关变化。 | 清理租户 code/id 缓存。 |
| `dictionary.updated` | 字典配置变化。 | 清理第三方/字典配置缓存。 |
| `third_party.updated` | OSS、文件安全、短信、极光、声网等第三方配置变化。 | 按 group 清理第三方配置缓存。 |
| `registration.config.updated` | 注册、短信开关变化。 | 清理注册配置缓存。 |
事件示例:
```json
{
"event_id": "evt_01K_config",
"trace_id": "req_01K",
"tenant_code": "default",
"source_service": "hotgo-admin",
"target_service": "im-server",
"event_type": "third_party.updated",
"biz_type": "config",
"biz_id": "im_third_party",
"occurred_at": "2026-08-26T10:05:00.000Z",
"payload": {
"tenant_code": "default",
"group": "im_third_party",
"keys": ["oss.endpoint", "file_security.enabled"]
}
}
```
禁言命令示例:
```json
{
"event_id": "evt_01K_mute",
"trace_id": "req_01K",
"tenant_code": "default",
"source_service": "hotgo-admin",
"target_service": "im-server",
"event_type": "group.member.mute",
"biz_type": "group",
"biz_id": "grp...",
"idempotency_key": "cmd_01K_mute",
"occurred_at": "2026-08-26T10:00:00.000Z",
"payload": {
"command_id": "cmd_01K_mute",
"actor_user_id": "usr_admin",
"topic": "grp...",
"user_id": "usr_target",
"reason": "刷屏",
"expires_at": null
}
}
```
### 11.7 IM Server 已接入的内部只读查询 API
为满足“Admin 不直连租户库”的边界,IM Server 提供内部只读查询接口给 HotGo/Admin 调用。接口默认关闭,通过 `internal_api.enabled=true` 开启。
请求统一鉴权头:
| Header | 说明 |
| --- | --- |
| `X-Service-Name` | 调用方服务名,如 `hotgo-admin`。 |
| `X-Timestamp` | Unix 秒级时间戳,默认 300 秒时间窗。 |
| `X-Nonce` | 一次性随机串,用于防重放。 |
| `X-Request-ID` | 调用链路 ID。 |
| `X-Signature` | HMAC-SHA256 签名,支持 hex 或 base64,可带 `sha256=` 前缀。 |
nonce 防重放优先使用 Redis 短状态 `SETNX + TTL`,Redis 未配置时退化为单节点进程内防重放;生产多节点必须配置 Redis。
签名明文:
```text
METHOD + "\n" +
PATH_WITH_QUERY + "\n" +
SHA256_HEX(BODY) + "\n" +
X-Timestamp + "\n" +
X-Nonce
```
当前首批接口:
| 接口 | 说明 |
| --- | --- |
| `GET /internal/admin-query/tenants/{tenant_code}/users/{user_id}` | 查询租户内用户基础资料。 |
| `GET /internal/admin-query/tenants/{tenant_code}/groups/{topic}` | 查询群资料和治理配置。 |
| `GET /internal/admin-query/tenants/{tenant_code}/groups/{topic}/members` | 查询群成员分页快照,支持 `limit/cursor/role/eligible_role/q`。 |
| `GET /internal/admin-query/tenants/{tenant_code}/messages/{topic}/{seq_id}` | 查询单条私聊或群聊消息,`seq_id` 为 `BIGINT`。 |
| `GET /internal/admin-query/tenants/{tenant_code}/files/{file_id}` | 查询文件元信息和存储位置。 |
配置示例:
```json
{
"internal_api": {
"enabled": true,
"time_window_seconds": 300,
"services": {
"hotgo-admin": {
"secret": "replace-with-random-secret",
"allow": ["admin_query"]
}
}
}
}
```
Docker 环境变量:
| 环境变量 | 说明 |
| --- | --- |
| `INTERNAL_API_ENABLED` | 是否开启内部接口,默认 `false`。 |
| `INTERNAL_API_TIME_WINDOW_SECONDS` | 签名时间窗。 |
| `INTERNAL_API_HOTGO_SERVICE_NAME` | Admin 服务名,默认 `hotgo-admin`。 |
| `INTERNAL_API_HOTGO_SECRET` | Admin 与 IM 共享签名密钥。 |
### 11.8 IM 访问 Admin 配置查询接口
为满足“IM 服务不直连 hotgo 库”的目标,IM Server 增加 Admin 内部配置查询客户端。该客户端默认关闭,开启后 `store.Tenants`、注册短信开关、第三方配置读取会优先请求 Admin 内部接口,失败时仍可回落旧数据库适配器,便于灰度迁移。
配置示例:
```json
{
"admin_internal_api": {
"enabled": true,
"base_url": "http://hotgo-admin:8080",
"service_name": "im-server",
"secret": "replace-with-random-secret",
"timeout_ms": 3000
}
}
```
接口约定:
| 接口 | 用途 |
| --- | --- |
| `GET /internal/im-config/tenants/by-code/{tenant_code}` | 按企业码获取租户路由信息,需返回 `id/code/name/state/db_name/db_dsn`。 |
| `GET /internal/im-config/tenants/by-id/{tenant_id}` | 按租户 ID 获取租户路由信息。 |
| `GET /internal/im-config/tenants?state=active&limit=1000` | 获取活跃租户列表,用于 Kafka Outbox 扫描。 |
| `GET /internal/im-config/registration/sms` | 获取注册短信开关,返回 `{"enabled": true}`。 |
| `GET /internal/im-config/third-party/{group}?keys=a,b` | 获取第三方配置,返回 `{"values": {"a": "..."}}`。 |
鉴权沿用内部接口 HMAC 方案,请求头为 `X-Service-Name`、`X-Timestamp`、`X-Nonce`、`X-Signature`;签名文本为 `METHOD + "\n" + RequestURI + "\n" + BodySHA256 + "\n" + Timestamp + "\n" + Nonce`。
Docker 环境变量:
| 环境变量 | 说明 |
| --- | --- |
| `ADMIN_INTERNAL_API_ENABLED` | 是否启用 IM 到 Admin 的配置查询客户端,默认 `false`。 |
| `ADMIN_INTERNAL_API_BASE_URL` | Admin 内部接口根地址。 |
| `ADMIN_INTERNAL_API_SERVICE_NAME` | IM 服务名,默认 `im-server`。 |
| `ADMIN_INTERNAL_API_SECRET` | Admin 与 IM 共享签名密钥。 |
| `ADMIN_INTERNAL_API_TIMEOUT_MS` | 请求超时时间。 |
### 11.9 IM Server 已接入的消息异步落库
IM Server 已接入 `im.message.persist`,用于把在线发送链路和最终消息落库解耦。
当前处理方式:
1. WS/HTTP 发送入口先同步完成权限校验、业务幂等、`seq_id` 分配和 Topic 游标占用。
2. Kafka 开启时,发送入口先写租户库 `im_kafka_outbox`,由 Outbox worker 可靠生产 `im.message.persist`,客户端仍立即拿到已分配的 `seq_id`。
3. Kafka 关闭时,继续走原同步落库路径,兼容本地和未接 Kafka 环境。
4. 消费者按 `message_kind` 区分私聊与群聊:
- `p2p` 写入 `im_messages`。
- `group` 写入 `im_group_messages`。
5. 消费侧复用 `im_kafka_consume_log` 做 `event_id + consumer_group` 幂等。
6. 失败按统一退避重试,超过上限写入 `im_kafka_dead_letters` 并投递 `im.dead.letter`。
### 11.10 IM Server 已接入的缓存失效消费
IM Server 已订阅 `im.cache.invalidate`,用于接收 Admin 或其他节点发出的通用缓存失效事件。消费侧复用 `im_kafka_consume_log` 做幂等,失败按统一退避重试,超过上限进入死信。
事件体示例:
```json
{
"event_id": "cache:tenant:default:1720000000",
"event_type": "cache.tenant.invalidate",
"tenant_code": "default",
"payload": {
"tenant_code": "default",
"tenant_id": 1,
"namespace": "tenant"
}
}
```
支持的失效范围:
| `event_type` / `namespace` | 处理方式 |
| --- | --- |
| `cache.tenant.invalidate` / `tenant` | 清理租户注册表与租户配置进程内缓存。 |
| `cache.registration.invalidate` / `registration` | 清理注册短信开关缓存。 |
| `cache.third_party.invalidate` / `third_party` | 按 `group` 清理第三方配置缓存。 |
| `cache.dictionary.invalidate` / `dictionary` | 按 `group` 清理字典/配置类缓存。 |
| `cache.redis_object.invalidate` / 任意 Redis `namespace` | 按 `keys` 删除用户、群、关系、权限、未读等 Redis 对象缓存。 |
| `cache.group_hot_messages.invalidate` / `group_hot_messages` | 按 `topic` 和可选 `ranges` 清理群热消息缓存。 |
### 11.11 IM Server 已接入的死信处理闭环
IM Server 已将死信从“只写表”补齐为“可查询、可忽略、可解决、可重放”的处理闭环。Admin 不直连租户库,列表和详情通过 IM 内部只读 API 查询;写操作仍通过 `im.admin.command` 投递给 IM。
Admin 查询接口:
| 接口 | 说明 |
| --- | --- |
| `GET /internal/admin-query/tenants/{tenant_code}/kafka/dead-letters?status=dead&limit=50&cursor=` | 死信分页列表,支持 `status/biz_type/biz_id/event_type` 过滤。 |
| `GET /internal/admin-query/tenants/{tenant_code}/kafka/dead-letters/{dead_id}` | 单条死信详情,包含原始 Kafka envelope。 |
Admin 处理命令:
| `event_type` | 说明 | Payload 关键字段 |
| --- | --- | --- |
| `kafka.dead_letter.replay` | 删除原消费日志并按原 `source_topic` 重投 envelope,成功后标记 `replayed`。 | `dead_id`、`actor_user_id`、`note` |
| `kafka.dead_letter.ignore` | 人工确认无需处理,标记 `ignored`。 | `dead_id`、`actor_user_id`、`note` |
| `kafka.dead_letter.resolve` | 线下修复后标记 `resolved`。 | `dead_id`、`actor_user_id`、`note` |
重放规则:
1. IM 先把 `im_kafka_dead_letters.status` 改为 `pending_replay`,并删除对应 `im_kafka_consume_log`。
2. IM 按死信记录中的 `source_topic` 重新投递原 envelope。
3. 投递成功后把死信标记为 `replayed`,并累计 `replay_count`。
4. 如果重放后仍失败,普通消费者会再次写入新的 dead 记录,保留完整处理痕迹。
### 11.12 IM Server 已接入的用户治理命令
IM Server 已在 `im.admin.command` 中接入用户治理类后台动作。Admin 侧写 hotgo 操作单并发送 Kafka Command,IM 消费后修改租户库用户状态、踢在线连接、清理缓存并回传 `im.admin.command.result`。
支持命令:
| `event_type` | 行为 | Payload 关键字段 |
| --- | --- | --- |
| `user.ban` | 封禁用户,设置 `StateSuspended` 并踢所有在线会话。 | `actor_user_id`、`user_id` 或 `target_user_id`、`reason` |
| `user.unban` | 解封用户,设置 `StateOK`。 | `actor_user_id`、`user_id` 或 `target_user_id`、`reason` |
| `user.freeze` | 冻结账号,设置 `StateSuspended` 并踢所有在线会话。 | `actor_user_id`、`user_id` 或 `target_user_id`、`reason` |
| `user.unfreeze` | 恢复账号,设置 `StateOK`。 | `actor_user_id`、`user_id` 或 `target_user_id`、`reason` |
| `user.device.kick` | 踢指定设备在线连接,并删除设备推送记录。 | `actor_user_id`、`user_id` 或 `target_user_id`、`device_id`、`reason` |
| `user.logout_all` | 踢用户所有在线连接,不修改账号状态。 | `actor_user_id`、`user_id` 或 `target_user_id`、`reason` |
| `user.account.cancel` | 注销账号,默认软删除;`hard=true` 时硬删除。 | `actor_user_id`、`user_id` 或 `target_user_id`、`hard`、`reason` |
| `user.account.delete` | 删除账号,按硬删除处理。 | `actor_user_id`、`user_id` 或 `target_user_id`、`reason` |
命令示例:
```json
{
"event_id": "admin:user:ban:default:usrabc:001",
"tenant_code": "default",
"source_service": "hotgo-admin",
"target_service": "im-server",
"event_type": "user.ban",
"biz_type": "user_governance",
"biz_id": "usrabc",
"idempotency_key": "admin-operation-id",
"occurred_at": "2026-08-26T10:10:00.000Z",
"payload": {
"command_id": "admin-operation-id",
"actor_user_id": "usrAdmin",
"user_id": "usrabc",
"reason": "违规处理"
}
}
```
处理说明:
1. 封禁/冻结共用 Tinode 用户状态 `StateSuspended`;解封/恢复共用 `StateOK`。
2. 状态变更后 IM 会清理用户缓存,并向本节点 Hub 投递用户状态变更。
3. 踢设备只影响匹配 `device_id` 的在线连接,同时删除该设备 Push token 记录。
4. 全端登出只踢在线连接,不修改用户状态。
5. 注销/删除会停止用户相关 Topic、清理缓存并调用 `store.Users.Delete`。
6. 命令执行成功后统一回传 `im.admin.command.result`,并写入 `im.audit.event`。
7. 群消息消费成功后继续写入 Redis 热消息缓存,便于 24 小时内热点拉取。
事件示例:
```json
{
"event_id": "msg.persist.1.grpabc.10086",
"tenant_code": "default",
"source_service": "im-server",
"target_service": "im-server",
"event_type": "message.persist.group",
"biz_type": "message",
"biz_id": "grpabc:10086",
"idempotency_key": "client-mid-001",
"version": 10086,
"occurred_at": "2026-08-26T10:10:00.000Z",
"actor": {
"type": "user",
"id": "usr..."
},
"payload": {
"message_kind": "group",
"read_by_sender": true,
"attachment_urls": [],
"message": {
"SeqId": 10086,
"Topic": "grpabc",
"From": "abcdef",
"Head": {
"x-im-client-mid": "client-mid-001"
},
"Content": {
"txt": "hello"
}
}
}
}
```
### 11.9 IM Server 已接入的 Kafka Outbox Worker
IM Server 已接入租户库 `im_kafka_outbox` 通用发送 worker,用于避免“业务库写成功但 Kafka 未发送”的事件丢失。
当前能力:
1. 启动后按 active 租户扫描每个租户库的 `im_kafka_outbox`。
2. 扫描 `pending/failed` 且已到 `next_retry_at` 的事件。
3. 发送成功后标记 `sent`、写入 `sent_at`。
4. 发送失败后标记 `failed`,按 Kafka 配置的 `retry_backoff_ms` 写入下一次重试时间。
5. 首个接入方为 `im.message.persist`,后续文件生命周期、工作台事件、审计事件可复用同一 outbox。
本轮暂不实现 `im.message.push` 离线 Push Kafka 化,已记录到 [IM 第一至十一章后续增强项](./IM-未实现后续增强项.md)。
### 11.10 IM Server 已接入的文件生命周期事件
IM Server 已接入 `im.file.lifecycle` 的首批生产与消费,用于把文件上传、附件绑定、群文件登记/删除纳入可靠事件链路。
当前事件:
| `event_type` | 生产时机 | 消费行为 |
| --- | --- | --- |
| `file.upload.completed` | HTTP/gRPC 上传成功并完成文件记录后。 | 当前作为可靠生命周期事实记录,文件安全任务仍由上传完成事务创建。 |
| `file.attachments.bind` | `im.message.persist` 消费成功后,如果消息携带附件。 | 私聊写 `im_filemsglinks`;群聊写 `im_group_files`。 |
| `group.file.created` | 群文件 API 登记成功后。 | 当前作为可靠生命周期事实记录,供后续 Admin 摘要、审计、回收策略复用。 |
| `group.file.deleted` | 群文件 API 删除成功后。 | 当前作为可靠生命周期事实记录,供后续 Admin 摘要、审计、回收策略复用。 |
说明:
1. Kafka 开启时,消息附件绑定从消息落库消费者中拆出,改由 `im.file.lifecycle` 消费者完成。
2. Kafka 关闭时,仍沿用原同步绑定路径,兼容本地环境。
3. 文件对象字节流仍走 OSS/文件服务,Kafka 不承载上传/下载内容。
4. 合规删除、临时文件 TTL 回收、OSS 删除动作仍保留为后续深化项。
### 11.11 IM Server 已接入的工作台事件
IM Server 已接入 `im.workbench.event` 的首批生产与消费。
当前生产范围:
| 模块 | 事件 |
| --- | --- |
| 会议 | `meeting.created`、`meeting.updated`、`meeting.canceled`、`meeting.reminded`、`meeting.response.updated` |
| 日报 | `daily_report.draft.created`、`daily_report.draft.saved`、`daily_report.submitted`、`daily_report.reminded` |
| 月报 | `monthly_report.draft.created`、`monthly_report.draft.saved`、`monthly_report.submitted`、`monthly_report.reopened` |
| 总经理信箱 | `mailbox.submitted` |
当前消费行为:
1. 消费 `im.workbench.event`。
2. 按 `target_user_ids` 生成系统通知。
3. 写入 `im_business_events_outbox`,由现有 WS 业务事件投递器下发在线客户端。
4. 使用 `im_kafka_consume_log` 做消费幂等,失败按统一重试和死信规则处理。
### 11.12 IM Server 已接入的审计事件
IM Server 已接入 `im.audit.event` 的可靠生产,审计事件写入租户库 `im_kafka_outbox` 后由 Kafka Outbox worker 发送。
当前审计范围:
| 类型 | 事件 |
| --- | --- |
| 工作台 | 会议、日报、月报、总经理信箱的状态变更。 |
| Admin Command | 后台群治理命令执行成功。 |
| Kafka 死信 | `kafka.dead_letter.created`。 |
| 文件生命周期 | 上传完成、附件绑定、群文件新增/删除。 |
审计事件当前不在 IM 内部落审计表,预期由 Admin/HotGo 消费 `im.audit.event` 后写入 `hotgo` 审计库或审计表。
## 12. 风险与约束
1. Kafka 是异步链路,Admin 后台命令不能默认立即成功,只能显示 `pending/running/succeeded/failed`。
2. 查询不能通过 Kafka 常规化实现,高频实时查询必须走内部只读 API 或 hotgo 摘要表。
3. 内部 API 不能暴露公网,签名、nonce、防重放和审计必须首批完成。
4. 配置事件可能丢失或延迟,IM 必须保留启动预热和缓存缺失时的 Admin 配置查询兜底。
5. Kafka 消费可能重复,所有消费者和业务写入都必须幂等。
6. Admin 和 IM 不跨库直连后,排障要依赖 `trace_id` 串联 hotgo 操作单、Kafka 事件、IM 执行日志和 WS 下发日志。
File diff suppressed because it is too large Load Diff
+90
View File
@@ -0,0 +1,90 @@
# IM Kafka 接入范围识别
> 日期:2026-08-26
> 依据:[IM 前后端逐图对接协议](./IM-前后端对接文档.md)
> 目标:从前后端对接文档逐章识别适合接入 Kafka 的链路,明确 Kafka 与 WebSocket、Redis、MySQL、OSS 的边界。
## 1. 总体边界
Kafka 主要用于跨系统异步事件、削峰落库、可靠重试、失败补偿、后台操作通知 C 端和审计流水;不替代 WebSocket 的在线实时连接,不替代 Redis 的短状态和热读缓存,也不替代 MySQL 的最终状态。
| 类型 | 建议承载 | 说明 |
| --- | --- | --- |
| 在线实时消息投递 | Redis Pub/Sub/Stream + WebSocket | 低延迟 fanout,节点收到后直接写 WS。 |
| 异步落库、重试、补偿 | Kafka | 适合消息持久化、通知、Push、审核、审计等可重放任务。 |
| 短状态、限流、验证码、在线状态 | Redis | TTL 明确,不需要 Kafka。 |
| 最终业务状态 | MySQL | Kafka 消费成功后写入或更新最终状态。 |
| 文件对象存储 | OSS | Kafka 只处理文件元数据、安全检测和生命周期事件。 |
## 2. 推荐 Topic
| Topic | 用途 | 典型生产方 | 典型消费方 |
| --- | --- | --- | --- |
| `im.message.persist` | 私聊、群聊消息异步落库 | IM WS 节点 | 消息持久化消费者 |
| `im.message.push` | 离线 Push 提醒 | 消息落库消费者、业务事件生产方 | 极光/厂商 Push 消费者 |
| `im.business.event` | 好友、群、会议、工作台等 C 端业务事件 | IM HTTP、Admin 后台 | IM 节点事件投递消费者 |
| `im.group.admin.event` | Admin 后台群操作投递到 C 端 | Admin 后台 | IM 节点 WS 投递消费者 |
| `im.cache.invalidate` | 跨节点缓存失效 | 业务写接口、后台配置 | IM 节点缓存消费者 |
| `im.file.security` | 文件内容安全和恶意文件检测 | 上传完成链路 | 阿里云安全检测消费者 |
| `im.file.lifecycle` | 孤立文件回收、引用绑定、合规删除 | 文件服务、业务写接口 | 文件生命周期消费者 |
| `im.audit.event` | 审计日志 | 所有敏感写链路 | 审计消费者 |
| `im.workbench.event` | 工作台模块红点、会议、日报、月报、信箱事件 | 工作台业务服务、Admin 后台 | IM WS 投递、系统通知消费者 |
| `im.dead.letter` | 失败事件归档 | 各消费者 | 运维补偿任务 |
## 3. 幂等与重试
Kafka 消息统一携带 `event_id`、`trace_id`、`tenant_code`、`biz_type`、`biz_id`、`action`、`version`、`occurred_at` 和 `payload`。消费端按 `event_id + consumer_group` 做幂等记录,业务表也必须保留唯一键兜底。
消息发送链路使用 `x-im-client-mid` 或 HTTP `Idempotency-Key` 作为业务幂等来源;Kafka 只负责可靠投递,不能作为唯一幂等判断。消费者失败按指数退避重试,超过上限写 `im.dead.letter` 或失败表,由后台补偿。
## 4. 逐章识别
| 章节 | 适合 Kafka 的范围 | 不建议 Kafka 的范围 |
| --- | --- | --- |
| 0. 公共对接约定 | 产品业务事件 outbox 可改为 Kafka 投递;消息异步落库、系统通知、离线 Push、跨节点缓存失效使用 Kafka;`x-im-client-mid` 幂等结果可作为生产 Kafka 前置校验。 | WebSocket 建连、`hi/login/sub/get/note` 本身不走 Kafka;在线状态继续 Redis。 |
| 1. 账号注册、登录与身份安全 | 企业注册申请、企业入口反馈、实名提交、密码重置成功、账号封禁/解封、管理员重置密码、审核结果通知、审计日志可走 Kafka。营业执照和反馈截图上传完成后投 `im.file.security`。 | 登录鉴权、验证码校验、Action Token、限流不走 Kafka;这些必须同步返回或走 Redis。 |
| 2. 消息首页、全局导航与系统通知 | 系统通知创建、已读异步统计、入口红点变更、离线 Push 提醒、消息摘要更新可走 Kafka。 | 首页查询、会话列表拉取、客户端断线重连流程不走 Kafka。 |
| 3. 单聊会话、会话建立与消息状态 | 私聊消息异步落库、好友申请/通过/拒绝/删除事件、P2P 订阅建立后的通知、媒体消息文件绑定事件、离线 Push 可走 Kafka。 | 在线 P2P 实时投递主链路不建议 Kafka;关系判断和发送权限校验必须同步完成。 |
| 4. 消息输入、媒体内容与消息操作 | @提醒 Push、消息撤回/编辑事件、收藏批量创建、转发任务、forward bundle 生成、消息搜索索引更新、附件 ACL 绑定可走 Kafka。 | 输入框、草稿、录音状态、上传进度和客户端 pending 队列不走 Kafka。 |
| 5. 通讯录、好友关系与黑名单 | 好友申请事件、好友关系变更、黑名单变更、二维码使用审计、资料快照更新、搜索索引更新、系统通知生成可走 Kafka。 | 好友列表查询、用户搜索同步结果、二维码解析鉴权不走 Kafka。 |
| 6. 群聊创建、资料维护与成员治理 | 群创建、成员加入/退出、邀请/审批、公告更新、资料更新、角色变更、禁言、群主转让、解散、群文件登记/删除、Admin 后台群操作投递到 C 端都适合 Kafka。Admin 后台写库后生产 `im.group.admin.event`,IM 节点消费后用 WS 下发给在线客户端。 | 群内在线消息 fanout 主链路继续 Redis + WS;群成员权限校验、成员数上限校验必须同步事务完成。 |
| 7. 文件下载、收藏管理与聊天记录检索 | 文件安全检测、恶意文件扫描、文件生命周期、收藏快照异步补全、收藏转发、搜索索引更新、合规删除事件可走 Kafka。文件下载本身不走 Kafka。 | OSS 上传/下载字节流、Range 续传、下载进度不走 Kafka;这些走文件服务和客户端任务管理。 |
| 8. 实时语音与视频通话 | 通话记录落库、未接来电 Push、通话结束审计、后台计费或统计、异常通话风控可走 Kafka。 | WebRTC offer/answer/ICE、接听、挂断等实时信令不建议 Kafka 主链路;仍走 WS。 |
| 9. 企业工作台与办公协同 | 工作台红点更新、会议创建/更新/取消/提醒/响应、日报/月报提交和催办、总经理信箱提交、工作台系统通知、后台配置变更、审计都适合 Kafka。 | 工作台模块列表、组织架构查询、报表详情查询不走 Kafka;普通查询走 MySQL/Redis 缓存。 |
| 10. 个人资料、通用设置、隐私与青少年模式 | 资料变更事件、隐私配置变更、青少年模式开启/关闭、多设备退出、Push Token 变更审计、App 发布配置缓存失效可走 Kafka。头像上传后走文件安全检测。 | 本地清缓存、系统权限引导、密码验证、Action Token 不走 Kafka。 |
| 11. 反诈安全、举报与意见反馈 | 风控确认审计、举报受理、证据文件安全检测、意见反馈、诊断包处理、后台审核通知、合规处置事件可走 Kafka。 | 风险弹窗同步拦截结果、举报原因查询、反馈配置查询不走 Kafka。 |
| 12. 联调实施清单 | Kafka 需要覆盖消息幂等、系统事件可信生产、文件安全检测、后台群操作到 C 端、离线 Push、失败重试和死信补偿验收。 | 前端基础联调请求不直接依赖 Kafka,除非该接口声明为异步任务。 |
## 5. 文件链路结论
文件上传、下载、Range、ETag、OSS 对象读写不走 Kafka。Kafka 只接入文件完成后的业务事件:
1. 上传完成后投递 `im.file.security`,调用阿里云内容安全和恶意文件检测。
2. 安全检测完成后更新文件审核状态,并通过 `im.business.event` 或系统通知告知相关用户。
3. 消息发送、收藏、群文件登记、日报/月报/反馈/举报提交成功后投递 `im.file.lifecycle`,建立业务引用和 ACL。
4. 业务撤回、删除、合规处置或临时文件 TTL 到期后投递 `im.file.lifecycle`,执行引用释放、禁用下载或 OSS 删除。
## 6. 工作台链路结论
工作台“查询类接口”不需要 Kafka,但工作台“状态变化和提醒类事件”适合 Kafka:
1. 会议创建、更新、取消、提醒、参会响应生产 `im.workbench.event`,消费者生成系统通知、红点并通过 WS 通知在线客户端。
2. 日报/月报提交、撤回、重开、管理员催办生产 `im.workbench.event`,用于红点、催办 Push 和审计。
3. 总经理信箱提交生产 `im.workbench.event` 和 `im.audit.event`,后台处理结果再回推 C 端。
4. 工作台模块配置变更生产 `im.cache.invalidate`,IM 节点刷新进程内配置缓存。
## 7. 已补充:缓存失效与 Admin 配置查询
1. `im.cache.invalidate` 已接入 IM 消费端,支持租户、注册短信、第三方配置、字典配置、Redis 对象缓存、群热消息缓存失效。
2. IM 已增加 Admin 内部配置查询客户端,开启 `admin_internal_api.enabled=true` 后优先通过 Admin 查询 hotgo 侧配置,旧数据库读取作为灰度兜底。
3. Admin 侧需要补齐 `/internal/im-config/*` 只读接口,并与 IM 使用同一套 HMAC 签名密钥。
4. Kafka 死信已补齐查询、忽略、解决、重放闭环:查询走 IM 内部 API,处理走 `im.admin.command`。
5. 用户治理类后台动作已接入 `im.admin.command`:封禁/解封、冻结/恢复、踢指定设备、全端登出、注销/删除账号。
## 8. 首批建议实现
1. 先实现 Kafka 基础生产者、消费者、配置、健康检查和消费幂等表。
2. 接入 `im.group.admin.event`,解决 Admin 后台群操作通过 Kafka 投递到 C 端。
3. 接入 `im.message.persist`,将消息异步落库和幂等重试闭环。
4. 接入 `im.file.security`,把 OSS 上传完成后的文件安全检测异步化。
5. 接入 `im.workbench.event`,先覆盖会议提醒、日报/月报催办、总经理信箱提交通知。
+12
View File
@@ -153,6 +153,7 @@
| P0 | 消息转发接口 | 对接文档有 `POST /v1/im/messages/forward`,当前未见路由;收藏转发不等同普通消息转发。 | 单条/多条消息可分别转发到好友、群和文件助手。 |
| P0 | 合并转发 Bundle | 对接文档有 `GET /v1/im/forward-bundles/{id}`,当前未见路由。 | 合并转发卡片可分页查看,权限隔离原会话。 |
| P0 | @ 实时通知与离线 Push | 当前消息元数据已落库,后续需要对被 @ 成员生成高优先级提醒和离线 Push。 | 被 @ 用户在线、离线、多端场景均收到一次提醒。 |
| P0 | 离线 Push Kafka 化 | `im.message.push` 已纳入 Kafka 方案,但本轮暂不实现;离线提醒仍按后续极光 Push 统一接入。 | 消息、@、通话、系统通知等离线场景统一投递 Kafka Push 任务,失败可重试和补偿。 |
| P0 | @ 全体频控 | @ 全体需要按租户、群、角色、时间窗口限频。 | 超限返回稳定错误码,管理员可配置限额。 |
| P1 | 群成员稳定游标分页 | 当前成员列表以基础 cursor 分页,后续应使用稳定游标避免成员变动跳页。 | 成员入群/退群时翻页不重复、不漏主要数据。 |
| P1 | @ 候选排序优化 | @ 面板候选应支持拼音首字母、最近互动、角色优先和本地缓存协同。 | 大群搜索响应稳定,候选排序符合产品规则。 |
@@ -351,3 +352,14 @@
4. 万人大群未实现增强项仅保留离线能力:离线只走极光 Push 提醒,客户端重连后按 Topic seq 拉取补偿;实时推送能力归入 `IM-Redis三层数据分层改造方案.md` 后续直接实现。
5. Redis 读性能深度优化继续保留:群成员分页 sorted set/稳定游标/主动失效、Topic 维度未读计数、bootstrap 聚合缓存和主动失效事件。
6. 每个增强项落地后同步更新本文档、`IM-系统设计文档.md`、`IM-前后端对接文档.md` 和接口审核报告,避免已完成项继续出现在待办清单。
## 13. Kafka 联调后续增强项
来源:`IM-Kafka接入联调测试文档.md` 10.2。本轮已完成后端 Kafka 主链路联调,但端到端页面与真实客户端仍需专项补测;同时 Kafka 断线恢复暴露出进程稳定性问题。
| 优先级 | 增强项 | 说明 | 建议验收点 |
| --- | --- | --- | --- |
| P0 | Admin Kafka 断线自动恢复 | 本地联调中停止 Kafka 后,Admin consumer 报 `client has run out of available brokers` 并 fatal 退出。 | Kafka 停止/恢复期间 Admin 进程不退出,consumer 退避重连,恢复后继续消费积压消息。 |
| P0 | IM Kafka consumer 自动恢复 | Kafka 恢复后 IM 仍 healthy,但 consumer 未立即消费新事件,重启 IM 后才补偿消费。 | Kafka broker 重启后无需重启 IM,`im.admin.command`、`im.message.persist`、`im.cache.invalidate` 等 Topic 自动恢复消费。 |
| P1 | Admin 前端页面人工验收 | 后端接口、Kafka、数据库状态已测;仍需在浏览器逐个菜单验收操作单、死信、审计和状态流转展示。 | 后台页面可展示 pending/running/succeeded/failed/dead,死信可查询、忽略、解决、重放,审计列表与详情可查看。 |
| P1 | 真实 C 端 WebSocket 联调 | 后台命令已生成业务事件/系统通知;仍需真实客户端在线观察 WS 实时刷新。 | 在线客户端收到群治理、用户治理、工作台、系统通知等事件;断线重连后可按 seq/bootstrap 补偿。 |
+81
View File
@@ -0,0 +1,81 @@
-- IM Kafka foundation tables for tenant databases.
-- Date: 2026-08-26
-- Scope: run once in every tenant business database, such as tinode.
-- Notes:
-- * Do not run in hotgo. Admin/HotGo side needs its own admin_command/outbox tables.
-- * Tables do not contain tenant_id because each tenant has an independent database.
CREATE TABLE IF NOT EXISTS im_kafka_consume_log (
id BIGINT NOT NULL AUTO_INCREMENT,
event_id VARCHAR(96) NOT NULL,
consumer_group VARCHAR(128) NOT NULL,
source_topic VARCHAR(128) NOT NULL,
event_type VARCHAR(96) NOT NULL DEFAULT '',
biz_type VARCHAR(64) NOT NULL DEFAULT '',
biz_id VARCHAR(128) NOT NULL DEFAULT '',
status VARCHAR(16) NOT NULL DEFAULT 'processing',
retry_count INT NOT NULL DEFAULT 0,
last_reason VARCHAR(64) NOT NULL DEFAULT '',
last_error VARCHAR(1000) NOT NULL DEFAULT '',
processing_deadline DATETIME(3),
succeeded_at DATETIME(3),
dead_at DATETIME(3),
created_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3),
updated_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3) ON UPDATE CURRENT_TIMESTAMP(3),
PRIMARY KEY(id),
UNIQUE INDEX uk_im_kafka_consume_log_event_group(event_id, consumer_group),
INDEX idx_im_kafka_consume_log_status(status, processing_deadline),
INDEX idx_im_kafka_consume_log_biz(biz_type, biz_id),
INDEX idx_im_kafka_consume_log_topic(source_topic, id)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
CREATE TABLE IF NOT EXISTS im_kafka_outbox (
id BIGINT NOT NULL AUTO_INCREMENT,
event_id VARCHAR(96) NOT NULL,
topic VARCHAR(128) NOT NULL,
event_type VARCHAR(96) NOT NULL,
biz_type VARCHAR(64) NOT NULL DEFAULT '',
biz_id VARCHAR(128) NOT NULL DEFAULT '',
key_value VARCHAR(256) NOT NULL DEFAULT '',
envelope JSON NOT NULL,
status VARCHAR(16) NOT NULL DEFAULT 'pending',
attempts INT NOT NULL DEFAULT 0,
next_retry_at DATETIME(3),
sent_at DATETIME(3),
last_reason VARCHAR(64) NOT NULL DEFAULT '',
last_error VARCHAR(1000) NOT NULL DEFAULT '',
created_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3),
updated_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3) ON UPDATE CURRENT_TIMESTAMP(3),
PRIMARY KEY(id),
UNIQUE INDEX uk_im_kafka_outbox_event(event_id),
INDEX idx_im_kafka_outbox_pending(status, next_retry_at, id),
INDEX idx_im_kafka_outbox_biz(biz_type, biz_id)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
CREATE TABLE IF NOT EXISTS im_kafka_dead_letters (
id BIGINT NOT NULL AUTO_INCREMENT,
dead_id VARCHAR(96) NOT NULL,
event_id VARCHAR(96) NOT NULL,
parent_event_id VARCHAR(96) NOT NULL DEFAULT '',
source_topic VARCHAR(128) NOT NULL,
consumer_group VARCHAR(128) NOT NULL,
event_type VARCHAR(96) NOT NULL DEFAULT '',
biz_type VARCHAR(64) NOT NULL DEFAULT '',
biz_id VARCHAR(128) NOT NULL DEFAULT '',
envelope JSON NOT NULL,
retry_count INT NOT NULL DEFAULT 0,
last_reason VARCHAR(64) NOT NULL DEFAULT '',
last_error VARCHAR(1000) NOT NULL DEFAULT '',
status VARCHAR(24) NOT NULL DEFAULT 'dead',
replay_count INT NOT NULL DEFAULT 0,
resolved_by VARCHAR(64) NOT NULL DEFAULT '',
resolved_at DATETIME(3),
resolve_note VARCHAR(1000) NOT NULL DEFAULT '',
created_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3),
updated_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3) ON UPDATE CURRENT_TIMESTAMP(3),
PRIMARY KEY(id),
UNIQUE INDEX uk_im_kafka_dead_letters_dead(dead_id),
INDEX idx_im_kafka_dead_letters_event(event_id),
INDEX idx_im_kafka_dead_letters_status(status, created_at),
INDEX idx_im_kafka_dead_letters_biz(biz_type, biz_id)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
+2
View File
@@ -78,8 +78,10 @@ require (
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/pierrec/lz4/v4 v4.1.15 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect
github.com/segmentio/kafka-go v0.4.49 // indirect
github.com/spiffe/go-spiffe/v2 v2.8.1 // indirect
github.com/stretchr/objx v0.5.0 // indirect
github.com/tjfoc/gmsm v1.4.1 // indirect
+4
View File
@@ -275,6 +275,8 @@ github.com/onsi/gomega v1.9.0/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoT
github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs=
github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc=
github.com/pierrec/lz4/v4 v4.1.15 h1:MO0/ucJhngq7299dKLwIMtgTfbkoSPF6AoMYDd8Q4q0=
github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo=
@@ -297,6 +299,8 @@ github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
github.com/segmentio/kafka-go v0.4.49 h1:GJiNX1d/g+kG6ljyJEoi9++PUMdXGAxb7JGPiDCuNmk=
github.com/segmentio/kafka-go v0.4.49/go.mod h1:Y1gn60kzLEEaW28YshXyk2+VCUKbJ3Qr6DrnT3i4+9E=
github.com/sirupsen/logrus v1.0.6/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc=
github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w=
github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g=
+260
View File
@@ -0,0 +1,260 @@
package main
import (
"bytes"
"crypto/hmac"
"crypto/rand"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"net/url"
"strconv"
"strings"
"time"
"github.com/tinode/chat/server/logs"
"github.com/tinode/chat/server/store"
"github.com/tinode/chat/server/store/types"
)
type adminInternalAPIConfig struct {
Enabled bool `json:"enabled"`
BaseURL string `json:"base_url"`
ServiceName string `json:"service_name"`
Secret string `json:"secret"`
TimeoutMS int `json:"timeout_ms"`
}
type adminInternalClient struct {
config adminInternalAPIConfig
http *http.Client
}
type adminInternalTenantDTO struct {
ID types.TenantID `json:"id"`
Code string `json:"code"`
Name string `json:"name"`
TenantDesc *string `json:"tenant_desc,omitempty"`
DBName string `json:"db_name"`
DBDSN string `json:"db_dsn"`
State types.TenantState `json:"state"`
CreatedAt time.Time `json:"created_at"`
CreatedBy int64 `json:"created_by"`
UpdatedAt time.Time `json:"updated_at"`
UpdatedBy int64 `json:"updated_by"`
}
func initAdminInternalAPI(raw json.RawMessage) error {
store.SetAdminInternalConfigProvider(nil)
if len(raw) == 0 || string(raw) == "null" {
return nil
}
var cfg adminInternalAPIConfig
if err := json.Unmarshal(raw, &cfg); err != nil {
return err
}
cfg.BaseURL = strings.TrimRight(strings.TrimSpace(cfg.BaseURL), "/")
cfg.ServiceName = strings.TrimSpace(cfg.ServiceName)
if cfg.ServiceName == "" {
cfg.ServiceName = "im-server"
}
if cfg.TimeoutMS <= 0 {
cfg.TimeoutMS = 3000
}
if !cfg.Enabled {
return nil
}
if cfg.BaseURL == "" || strings.TrimSpace(cfg.Secret) == "" {
return errors.New("admin_internal_api enabled but base_url or secret is empty")
}
store.SetAdminInternalConfigProvider(&adminInternalClient{
config: cfg,
http: &http.Client{Timeout: time.Duration(cfg.TimeoutMS) * time.Millisecond},
})
logs.Info.Printf("Admin internal config client enabled: base_url=%s service=%s", cfg.BaseURL, cfg.ServiceName)
return nil
}
func (c *adminInternalClient) GetTenantByCode(code string) (*types.Tenant, error) {
var tenant adminInternalTenantDTO
if err := c.getJSON("/internal/im-config/tenants/by-code/"+url.PathEscape(strings.TrimSpace(code)), nil, &tenant); err != nil {
return nil, err
}
return tenant.toStoreTenant(), nil
}
func (c *adminInternalClient) GetTenantByID(id types.TenantID) (*types.Tenant, error) {
if !id.IsValid() {
return nil, types.ErrMalformed
}
var tenant adminInternalTenantDTO
if err := c.getJSON("/internal/im-config/tenants/by-id/"+strconv.FormatInt(int64(id), 10), nil, &tenant); err != nil {
return nil, err
}
return tenant.toStoreTenant(), nil
}
func (c *adminInternalClient) ListActiveTenants(limit int) ([]types.Tenant, error) {
query := url.Values{}
query.Set("state", "active")
if limit > 0 {
query.Set("limit", strconv.Itoa(limit))
}
var wrapper struct {
Items []adminInternalTenantDTO `json:"items"`
}
if err := c.getJSON("/internal/im-config/tenants", query, &wrapper); err != nil {
return nil, err
}
tenants := make([]types.Tenant, 0, len(wrapper.Items))
for _, item := range wrapper.Items {
if tenant := item.toStoreTenant(); tenant != nil {
tenants = append(tenants, *tenant)
}
}
return tenants, nil
}
func (c *adminInternalClient) RegistrationSMSEnabled() (bool, error) {
var wrapper struct {
Enabled bool `json:"enabled"`
}
if err := c.getJSON("/internal/im-config/registration/sms", nil, &wrapper); err != nil {
return true, err
}
return wrapper.Enabled, nil
}
func (c *adminInternalClient) ThirdPartyConfig(group string, keys ...string) (map[string]string, error) {
query := url.Values{}
normalized := make([]string, 0, len(keys))
for _, key := range keys {
if key = strings.TrimSpace(key); key != "" {
normalized = append(normalized, key)
}
}
if len(normalized) > 0 {
query.Set("keys", strings.Join(normalized, ","))
}
var wrapper struct {
Values map[string]string `json:"values"`
}
if err := c.getJSON("/internal/im-config/third-party/"+url.PathEscape(strings.TrimSpace(group)), query, &wrapper); err != nil {
return nil, err
}
if wrapper.Values == nil {
wrapper.Values = map[string]string{}
}
return wrapper.Values, nil
}
func (c *adminInternalClient) getJSON(path string, query url.Values, out any) error {
if c == nil || c.http == nil {
return types.ErrUnsupported
}
path = "/" + strings.TrimLeft(path, "/")
u, err := url.Parse(c.config.BaseURL + path)
if err != nil {
return err
}
if len(query) > 0 {
u.RawQuery = query.Encode()
}
req, err := http.NewRequest(http.MethodGet, u.String(), nil)
if err != nil {
return err
}
if err = c.signRequest(req, nil); err != nil {
return err
}
resp, err := c.http.Do(req)
if err != nil {
return err
}
defer resp.Body.Close()
body, err := io.ReadAll(io.LimitReader(resp.Body, 4<<20))
if err != nil {
return err
}
if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices {
return fmt.Errorf("admin internal api status=%d body=%s", resp.StatusCode, string(body))
}
var parsed struct {
Code int `json:"code"`
Data json.RawMessage `json:"data"`
}
if err = json.Unmarshal(body, &parsed); err != nil {
return err
}
if parsed.Code != 0 {
return fmt.Errorf("admin internal api code=%d", parsed.Code)
}
if out == nil || len(parsed.Data) == 0 || string(parsed.Data) == "null" {
return nil
}
return json.Unmarshal(parsed.Data, out)
}
func (c *adminInternalClient) signRequest(req *http.Request, body []byte) error {
nonce, err := randomInternalNonce()
if err != nil {
return err
}
timestamp := strconv.FormatInt(time.Now().Unix(), 10)
bodyHash := sha256.Sum256(body)
signingText := strings.Join([]string{
req.Method,
req.URL.RequestURI(),
hex.EncodeToString(bodyHash[:]),
timestamp,
nonce,
}, "\n")
req.Header.Set(internalHeaderService, c.config.ServiceName)
req.Header.Set(internalHeaderTimestamp, timestamp)
req.Header.Set(internalHeaderNonce, nonce)
req.Header.Set(internalHeaderSignature, signInternalRequest(c.config.Secret, signingText))
req.Header.Set("Accept", "application/json")
if len(body) > 0 {
req.Body = io.NopCloser(bytes.NewReader(body))
req.ContentLength = int64(len(body))
req.Header.Set("Content-Type", "application/json")
}
return nil
}
func randomInternalNonce() (string, error) {
var raw [16]byte
if _, err := rand.Read(raw[:]); err != nil {
return "", err
}
return hex.EncodeToString(raw[:]), nil
}
func signInternalRequest(secret, signingText string) string {
mac := hmac.New(sha256.New, []byte(secret))
_, _ = mac.Write([]byte(signingText))
return hex.EncodeToString(mac.Sum(nil))
}
func (t adminInternalTenantDTO) toStoreTenant() *types.Tenant {
if t.ID.IsZero() && strings.TrimSpace(t.Code) == "" {
return nil
}
return &types.Tenant{
ID: t.ID,
Code: strings.TrimSpace(t.Code),
Name: t.Name,
TenantDesc: t.TenantDesc,
DBName: t.DBName,
DBDSN: t.DBDSN,
State: t.State,
CreatedAt: t.CreatedAt,
CreatedBy: t.CreatedBy,
UpdatedAt: t.UpdatedAt,
UpdatedBy: t.UpdatedBy,
}
}
+18
View File
@@ -214,6 +214,7 @@ type Adapter interface {
type TenantAdapter interface {
TenantGetByCode(code string) (*t.Tenant, error)
TenantGetByID(id t.TenantID) (*t.Tenant, error)
TenantListActive(limit int) ([]t.Tenant, error)
}
// RegistrationConfigAdapter reads global registration policy from the control database.
@@ -234,6 +235,11 @@ type TenantEntryAdapter interface {
TenantEntryFeedbackCreate(*t.TenantEntryFeedback) (*t.TenantEntryFeedback, error)
}
type TenantInternalAdminQueryAdapter interface {
TenantInternalAdminQuery(tenantID t.TenantID, query string, countQuery string, args []any, page, perPage int) (*t.InternalAdminQueryResult, error)
TenantInternalAdminQueryOne(tenantID t.TenantID, query string, args []any) (map[string]any, error)
}
// TenantRoutedAdapter can return a tenant-specific adapter, e.g. for the
// "one tenant, one database" deployment model.
type TenantRoutedAdapter interface {
@@ -249,6 +255,18 @@ type TenantHTTPInfrastructureAdapter interface {
TenantBusinessEventCreate(t.TenantID, *t.BusinessEvent) error
TenantBusinessEventListPending(t.TenantID, int, time.Time) ([]t.BusinessEvent, error)
TenantBusinessEventMarkDelivered(t.TenantID, string, time.Time) error
TenantKafkaConsumeBegin(t.TenantID, *t.KafkaConsumeRecord, time.Time) (*t.KafkaConsumeRecord, bool, error)
TenantKafkaConsumeSucceed(t.TenantID, string, string, time.Time) error
TenantKafkaConsumeFail(t.TenantID, string, string, string, string, time.Time) (*t.KafkaConsumeRecord, error)
TenantKafkaConsumeDead(t.TenantID, *t.KafkaDeadLetter, time.Time) error
TenantKafkaDeadLetterList(t.TenantID, t.KafkaDeadLetterListOptions) (*t.KafkaDeadLetterPage, error)
TenantKafkaDeadLetterGet(t.TenantID, string) (*t.KafkaDeadLetter, error)
TenantKafkaDeadLetterMark(t.TenantID, string, string, string, string, time.Time) error
TenantKafkaDeadLetterMarkReplay(t.TenantID, string, string, string, time.Time) (*t.KafkaDeadLetter, error)
TenantKafkaOutboxCreate(t.TenantID, *t.KafkaOutboxRecord) error
TenantKafkaOutboxListPending(t.TenantID, int, time.Time) ([]t.KafkaOutboxRecord, error)
TenantKafkaOutboxMarkSent(t.TenantID, string, time.Time) error
TenantKafkaOutboxMarkFailed(t.TenantID, string, string, string, time.Time, time.Time) error
}
// TenantRegistrationInviteAdapter stores invitation definitions in the
+25
View File
@@ -428,6 +428,31 @@ func (a *adapter) TenantGetByID(id t.TenantID) (*t.Tenant, error) {
return a.tenantByID(id)
}
func (a *adapter) TenantListActive(limit int) ([]t.Tenant, error) {
if limit <= 0 {
limit = 1000
}
ctx, cancel := a.getContext()
if cancel != nil {
defer cancel()
}
db := a.db
if a.registryDB != nil {
db = a.registryDB
}
var tenants []t.Tenant
err := db.SelectContext(ctx, &tenants,
`SELECT id, code, name, tenant_desc, IFNULL(db_name,'') AS db_name, IFNULL(db_dsn,'') AS db_dsn,
state, created_at, created_by, updated_at, updated_by
FROM im_tenant WHERE state=? ORDER BY id ASC LIMIT ?`, t.TenantStateActive, limit)
if isUnknownColumn(err) {
err = db.SelectContext(ctx, &tenants,
`SELECT id, code, name, tenant_desc, state, created_at, created_by, updated_at, updated_by
FROM im_tenant WHERE state=? ORDER BY id ASC LIMIT ?`, t.TenantStateActive, limit)
}
return tenants, err
}
func (a *adapter) TenantAdapterFor(tenantID t.TenantID) (dbadapter.TenantBusinessAdapter, dbadapter.TenantAuthAdapter, error) {
if !a.tenantDBRouting {
return a, a, nil
+46
View File
@@ -29,6 +29,52 @@ func ensureIMHTTPInfrastructureTables(exec sqlExecutor) error {
INDEX idx_im_business_events_outbox_pending(status, next_retry_at, id),
INDEX idx_im_business_events_outbox_target(target_type, target, id)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci`,
`CREATE TABLE IF NOT EXISTS im_kafka_consume_log (
id BIGINT NOT NULL AUTO_INCREMENT, event_id VARCHAR(96) NOT NULL,
consumer_group VARCHAR(128) NOT NULL, source_topic VARCHAR(128) NOT NULL,
event_type VARCHAR(96) NOT NULL DEFAULT '', biz_type VARCHAR(64) NOT NULL DEFAULT '',
biz_id VARCHAR(128) NOT NULL DEFAULT '', status VARCHAR(16) NOT NULL DEFAULT 'processing',
retry_count INT NOT NULL DEFAULT 0, last_reason VARCHAR(64) NOT NULL DEFAULT '',
last_error VARCHAR(1000) NOT NULL DEFAULT '', processing_deadline DATETIME(3),
succeeded_at DATETIME(3), dead_at DATETIME(3),
created_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3),
updated_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3) ON UPDATE CURRENT_TIMESTAMP(3),
PRIMARY KEY(id), UNIQUE INDEX uk_im_kafka_consume_log_event_group(event_id, consumer_group),
INDEX idx_im_kafka_consume_log_status(status, processing_deadline),
INDEX idx_im_kafka_consume_log_biz(biz_type, biz_id),
INDEX idx_im_kafka_consume_log_topic(source_topic, id)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci`,
`CREATE TABLE IF NOT EXISTS im_kafka_outbox (
id BIGINT NOT NULL AUTO_INCREMENT, event_id VARCHAR(96) NOT NULL,
topic VARCHAR(128) NOT NULL, event_type VARCHAR(96) NOT NULL,
biz_type VARCHAR(64) NOT NULL DEFAULT '', biz_id VARCHAR(128) NOT NULL DEFAULT '',
key_value VARCHAR(256) NOT NULL DEFAULT '', envelope JSON NOT NULL,
status VARCHAR(16) NOT NULL DEFAULT 'pending', attempts INT NOT NULL DEFAULT 0,
next_retry_at DATETIME(3), sent_at DATETIME(3), last_reason VARCHAR(64) NOT NULL DEFAULT '',
last_error VARCHAR(1000) NOT NULL DEFAULT '',
created_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3),
updated_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3) ON UPDATE CURRENT_TIMESTAMP(3),
PRIMARY KEY(id), UNIQUE INDEX uk_im_kafka_outbox_event(event_id),
INDEX idx_im_kafka_outbox_pending(status, next_retry_at, id),
INDEX idx_im_kafka_outbox_biz(biz_type, biz_id)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci`,
`CREATE TABLE IF NOT EXISTS im_kafka_dead_letters (
id BIGINT NOT NULL AUTO_INCREMENT, dead_id VARCHAR(96) NOT NULL,
event_id VARCHAR(96) NOT NULL, parent_event_id VARCHAR(96) NOT NULL DEFAULT '',
source_topic VARCHAR(128) NOT NULL, consumer_group VARCHAR(128) NOT NULL,
event_type VARCHAR(96) NOT NULL DEFAULT '', biz_type VARCHAR(64) NOT NULL DEFAULT '',
biz_id VARCHAR(128) NOT NULL DEFAULT '', envelope JSON NOT NULL,
retry_count INT NOT NULL DEFAULT 0, last_reason VARCHAR(64) NOT NULL DEFAULT '',
last_error VARCHAR(1000) NOT NULL DEFAULT '', status VARCHAR(24) NOT NULL DEFAULT 'dead',
replay_count INT NOT NULL DEFAULT 0, resolved_by VARCHAR(64) NOT NULL DEFAULT '',
resolved_at DATETIME(3), resolve_note VARCHAR(1000) NOT NULL DEFAULT '',
created_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3),
updated_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3) ON UPDATE CURRENT_TIMESTAMP(3),
PRIMARY KEY(id), UNIQUE INDEX uk_im_kafka_dead_letters_dead(dead_id),
INDEX idx_im_kafka_dead_letters_event(event_id),
INDEX idx_im_kafka_dead_letters_status(status, created_at),
INDEX idx_im_kafka_dead_letters_biz(biz_type, biz_id)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci`,
`CREATE TABLE IF NOT EXISTS im_realname_appeals (
id BIGINT NOT NULL, appeal_id VARCHAR(64) NOT NULL, user_id BIGINT NOT NULL,
reason VARCHAR(500) NOT NULL DEFAULT '', attachments_json JSON,
@@ -12,13 +12,16 @@ func TestEnsureIMHTTPInfrastructureTablesUsesSelectedTenantDatabase(t *testing.T
if err := ensureIMHTTPInfrastructureTables(exec); err != nil {
t.Fatal(err)
}
if len(exec.statements) != 5 {
t.Fatalf("created %d infrastructure tables, want 5", len(exec.statements))
if len(exec.statements) != 8 {
t.Fatalf("created %d infrastructure tables, want 8", len(exec.statements))
}
joined := strings.ToLower(strings.Join(exec.statements, "\n"))
for _, table := range []string{
"im_http_idempotency",
"im_business_events_outbox",
"im_kafka_consume_log",
"im_kafka_outbox",
"im_kafka_dead_letters",
"im_realname_appeals",
"im_conversation_settings",
"im_topic_drafts",
+64
View File
@@ -2289,6 +2289,9 @@ func (a *adapter) TenantMessageSearch(id t.TenantID, topic string, uid t.Uid, qu
if limit <= 0 || limit > 100 {
limit = 50
}
if t.GetTopicCat(topic) == t.TopicCatGrp {
return a.tenantGroupMessageSearch(id, topic, uid, queryText, messageType, cursor, limit)
}
where := []string{"m.topic=?", "m.delid=0"}
args := []any{topic}
if cursor != "" {
@@ -2347,6 +2350,67 @@ func (a *adapter) TenantMessageSearch(id t.TenantID, topic string, uid t.Uid, qu
return msgs, nextCursor, nil
}
func (a *adapter) tenantGroupMessageSearch(id t.TenantID, topic string, uid t.Uid, queryText, messageType, cursor string, limit int) ([]t.Message, string, error) {
where := []string{"m.status!='deleted'", "m.topic=?"}
args := []any{topic}
if cursor != "" {
parsed, err := strconv.Atoi(cursor)
if err != nil || parsed <= 0 {
return nil, "", t.ErrMalformed
}
where = append(where, "m.seqid<?")
args = append(args, parsed)
}
queryText = strings.TrimSpace(queryText)
if queryText != "" {
where = append(where, "CAST(m.content AS CHAR) LIKE ?")
args = append(args, "%"+queryText+"%")
}
messageType = strings.TrimSpace(messageType)
if messageType != "" && messageType != "all" {
where = append(where, "m.msg_type=?")
args = append(args, messageType)
}
ctx, cancel := a.getContext()
if cancel != nil {
defer cancel()
}
queryArgs := append([]any{store.DecodeUid(uid)}, args...)
queryArgs = append(queryArgs, limit+1)
rows, err := a.db.QueryxContext(ctx,
"SELECT m.id,m.created_at AS createdat,m.updated_at AS updatedat,m.deleted_at AS deletedat,0 AS delid,m.seqid,m.topic,m.from_user AS `from`,IFNULL(m.client_msg_id,'') AS client_msg_id,m.version,m.recalled_at,m.recalled_by,m.head,m.content "+
"FROM im_group_messages AS m LEFT JOIN im_dellog AS d "+
"ON d.topic COLLATE utf8mb4_unicode_ci=m.topic AND m.seqid BETWEEN d.low AND d.hi-1 AND d.deletedfor=? "+
"WHERE "+strings.Join(where, " AND ")+" AND d.deletedfor IS NULL ORDER BY m.seqid DESC LIMIT ?",
queryArgs...)
if err != nil {
return nil, "", err
}
defer rows.Close()
msgs := make([]t.Message, 0, limit)
nextCursor := ""
for rows.Next() {
row := struct {
t.Message
RecalledBy sql.NullInt64 `db:"recalled_by"`
}{}
if err = rows.StructScan(&row); err != nil {
return nil, "", err
}
msg := normalizeGroupMessage(id, row.Message, row.RecalledBy)
msg.Head = kvMapFromJSON(msg.Head)
if len(msgs) == limit {
nextCursor = strconv.FormatInt(msg.SeqId, 10)
continue
}
msgs = append(msgs, msg)
}
if err = rows.Err(); err != nil {
return nil, "", err
}
return msgs, nextCursor, nil
}
func (a *adapter) TenantCallCreate(id t.TenantID, record *t.CallRecord) error {
if !a.tenantScopedDB {
tenantAdp, err := a.routedTenantAdapter(id)
+509
View File
@@ -4,8 +4,11 @@
package mysql
import (
"context"
"database/sql"
"encoding/json"
"strconv"
"strings"
"time"
"github.com/jmoiron/sqlx"
@@ -225,6 +228,512 @@ func (a *adapter) TenantBusinessEventMarkDelivered(id t.TenantID, eventID string
return nil
}
func (a *adapter) TenantKafkaConsumeBegin(id t.TenantID, record *t.KafkaConsumeRecord, deadline time.Time) (*t.KafkaConsumeRecord, bool, error) {
tenantAdp, err := a.infrastructureAdapter(id)
if err != nil {
return nil, false, err
}
if record == nil || record.EventID == "" || record.ConsumerGroup == "" || record.SourceTopic == "" {
return nil, false, t.ErrMalformed
}
if tenantAdp != a {
return tenantAdp.TenantKafkaConsumeBegin(id, record, deadline)
}
now := t.TimeNow()
_, err = a.db.Exec(`INSERT INTO im_kafka_consume_log
(event_id,consumer_group,source_topic,event_type,biz_type,biz_id,status,processing_deadline,created_at,updated_at)
VALUES(?,?,?,?,?,?,?,?,?,?)`, record.EventID, record.ConsumerGroup, record.SourceTopic, record.EventType,
record.BizType, record.BizID, t.KafkaConsumeProcessing, deadline, now, now)
if err == nil {
record.Status = t.KafkaConsumeProcessing
record.ProcessingDeadline = &deadline
record.CreatedAt = now
record.UpdatedAt = now
return record, true, nil
}
if !isDupe(err) {
return nil, false, err
}
existing, err := a.kafkaConsumeGet(record.EventID, record.ConsumerGroup)
if err != nil {
return nil, false, err
}
if existing.Status == t.KafkaConsumeSucceeded || existing.Status == t.KafkaConsumeDead {
return existing, false, nil
}
if existing.Status == t.KafkaConsumeProcessing && existing.ProcessingDeadline != nil && existing.ProcessingDeadline.After(now) {
return existing, false, nil
}
res, err := a.db.Exec(`UPDATE im_kafka_consume_log SET status=?,source_topic=?,event_type=?,biz_type=?,biz_id=?,
processing_deadline=?,updated_at=? WHERE event_id=? AND consumer_group=? AND status<>?`,
t.KafkaConsumeProcessing, record.SourceTopic, record.EventType, record.BizType, record.BizID,
deadline, now, record.EventID, record.ConsumerGroup, t.KafkaConsumeSucceeded)
if err != nil {
return nil, false, err
}
if affected, _ := res.RowsAffected(); affected == 0 {
current, getErr := a.kafkaConsumeGet(record.EventID, record.ConsumerGroup)
return current, false, getErr
}
current, err := a.kafkaConsumeGet(record.EventID, record.ConsumerGroup)
return current, err == nil, err
}
func (a *adapter) TenantKafkaConsumeSucceed(id t.TenantID, eventID, consumerGroup string, now time.Time) error {
tenantAdp, err := a.infrastructureAdapter(id)
if err != nil {
return err
}
if eventID == "" || consumerGroup == "" {
return t.ErrMalformed
}
if tenantAdp != a {
return tenantAdp.TenantKafkaConsumeSucceed(id, eventID, consumerGroup, now)
}
res, err := a.db.Exec(`UPDATE im_kafka_consume_log SET status=?,succeeded_at=?,processing_deadline=NULL,updated_at=?
WHERE event_id=? AND consumer_group=?`, t.KafkaConsumeSucceeded, now, now, eventID, consumerGroup)
if err != nil {
return err
}
if affected, _ := res.RowsAffected(); affected == 0 {
return t.ErrNotFound
}
return nil
}
func (a *adapter) TenantKafkaConsumeFail(id t.TenantID, eventID, consumerGroup, reason, message string, now time.Time) (*t.KafkaConsumeRecord, error) {
tenantAdp, err := a.infrastructureAdapter(id)
if err != nil {
return nil, err
}
if eventID == "" || consumerGroup == "" {
return nil, t.ErrMalformed
}
if tenantAdp != a {
return tenantAdp.TenantKafkaConsumeFail(id, eventID, consumerGroup, reason, message, now)
}
_, err = a.db.Exec(`UPDATE im_kafka_consume_log SET status=?,retry_count=retry_count+1,last_reason=?,
last_error=?,processing_deadline=NULL,updated_at=? WHERE event_id=? AND consumer_group=?`,
t.KafkaConsumeFailed, truncateKafkaString(reason, 64), truncateKafkaString(message, 1000), now, eventID, consumerGroup)
if err != nil {
return nil, err
}
return a.kafkaConsumeGet(eventID, consumerGroup)
}
func (a *adapter) TenantKafkaConsumeDead(id t.TenantID, dead *t.KafkaDeadLetter, now time.Time) error {
tenantAdp, err := a.infrastructureAdapter(id)
if err != nil {
return err
}
if dead == nil || dead.DeadID == "" || dead.EventID == "" || dead.ConsumerGroup == "" || dead.SourceTopic == "" || len(dead.Envelope) == 0 {
return t.ErrMalformed
}
if tenantAdp != a {
return tenantAdp.TenantKafkaConsumeDead(id, dead, now)
}
tx, err := a.db.BeginTxx(context.Background(), nil)
if err != nil {
return err
}
defer tx.Rollback()
if _, err = tx.Exec(`INSERT INTO im_kafka_dead_letters
(dead_id,event_id,parent_event_id,source_topic,consumer_group,event_type,biz_type,biz_id,envelope,
retry_count,last_reason,last_error,status,replay_count,created_at,updated_at)
VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)
ON DUPLICATE KEY UPDATE last_reason=VALUES(last_reason),last_error=VALUES(last_error),updated_at=VALUES(updated_at)`,
dead.DeadID, dead.EventID, dead.ParentEventID, dead.SourceTopic, dead.ConsumerGroup, dead.EventType,
dead.BizType, dead.BizID, dead.Envelope, dead.RetryCount, truncateKafkaString(dead.LastReason, 64),
truncateKafkaString(dead.LastError, 1000), t.KafkaDeadStatusDead, dead.ReplayCount, now, now); err != nil {
return err
}
if _, err = tx.Exec(`UPDATE im_kafka_consume_log SET status=?,dead_at=?,processing_deadline=NULL,last_reason=?,
last_error=?,updated_at=? WHERE event_id=? AND consumer_group=?`,
t.KafkaConsumeDead, now, truncateKafkaString(dead.LastReason, 64), truncateKafkaString(dead.LastError, 1000),
now, dead.EventID, dead.ConsumerGroup); err != nil {
return err
}
return tx.Commit()
}
func (a *adapter) TenantKafkaDeadLetterList(id t.TenantID, opts t.KafkaDeadLetterListOptions) (*t.KafkaDeadLetterPage, error) {
tenantAdp, err := a.infrastructureAdapter(id)
if err != nil {
return nil, err
}
if tenantAdp != a {
return tenantAdp.TenantKafkaDeadLetterList(id, opts)
}
limit := opts.Limit
if limit <= 0 || limit > 200 {
limit = 50
}
where := []string{"1=1"}
args := []any{}
if opts.Status != "" {
where = append(where, "status=?")
args = append(args, opts.Status)
}
if opts.BizType != "" {
where = append(where, "biz_type=?")
args = append(args, opts.BizType)
}
if opts.BizID != "" {
where = append(where, "biz_id=?")
args = append(args, opts.BizID)
}
if opts.EventType != "" {
where = append(where, "event_type=?")
args = append(args, opts.EventType)
}
if opts.Cursor != "" {
where = append(where, "id<?")
args = append(args, opts.Cursor)
}
args = append(args, limit+1)
rows, err := a.db.Queryx(`SELECT id,dead_id,event_id,parent_event_id,source_topic,consumer_group,event_type,biz_type,biz_id,
envelope,retry_count,last_reason,last_error,status,replay_count,created_at,updated_at
FROM im_kafka_dead_letters WHERE `+strings.Join(where, " AND ")+` ORDER BY id DESC LIMIT ?`, args...)
if err != nil {
return nil, err
}
defer rows.Close()
page := &t.KafkaDeadLetterPage{Items: make([]t.KafkaDeadLetter, 0, limit)}
for rows.Next() {
dead, err := scanKafkaDeadLetter(rows)
if err != nil {
return nil, err
}
if len(page.Items) == limit {
page.NextCursor = strconv.FormatInt(dead.ID, 10)
continue
}
page.Items = append(page.Items, *dead)
}
return page, rows.Err()
}
func (a *adapter) TenantKafkaDeadLetterGet(id t.TenantID, deadID string) (*t.KafkaDeadLetter, error) {
tenantAdp, err := a.infrastructureAdapter(id)
if err != nil {
return nil, err
}
if strings.TrimSpace(deadID) == "" {
return nil, t.ErrMalformed
}
if tenantAdp != a {
return tenantAdp.TenantKafkaDeadLetterGet(id, deadID)
}
rows, err := a.db.Queryx(`SELECT id,dead_id,event_id,parent_event_id,source_topic,consumer_group,event_type,biz_type,biz_id,
envelope,retry_count,last_reason,last_error,status,replay_count,created_at,updated_at
FROM im_kafka_dead_letters WHERE dead_id=?`, strings.TrimSpace(deadID))
if err != nil {
return nil, err
}
defer rows.Close()
if rows.Next() {
return scanKafkaDeadLetter(rows)
}
if err = rows.Err(); err != nil {
return nil, err
}
return nil, t.ErrNotFound
}
func (a *adapter) TenantKafkaDeadLetterMark(id t.TenantID, deadID, status, resolvedBy, note string, now time.Time) error {
tenantAdp, err := a.infrastructureAdapter(id)
if err != nil {
return err
}
deadID = strings.TrimSpace(deadID)
status = strings.TrimSpace(status)
if deadID == "" || (status != t.KafkaDeadStatusIgnored && status != t.KafkaDeadStatusResolved && status != t.KafkaDeadStatusReplayed) {
return t.ErrMalformed
}
if tenantAdp != a {
return tenantAdp.TenantKafkaDeadLetterMark(id, deadID, status, resolvedBy, note, now)
}
res, err := a.db.Exec(`UPDATE im_kafka_dead_letters SET status=?,resolved_by=?,resolved_at=?,resolve_note=?,updated_at=?
WHERE dead_id=? AND status IN (?,?,?)`, status, truncateKafkaString(resolvedBy, 64), now,
truncateKafkaString(note, 1000), now, deadID, t.KafkaDeadStatusDead, t.KafkaDeadStatusPendingReplay, t.KafkaDeadStatusReplayed)
if err != nil {
return err
}
if affected, _ := res.RowsAffected(); affected == 0 {
return t.ErrNotFound
}
return nil
}
func (a *adapter) TenantKafkaDeadLetterMarkReplay(id t.TenantID, deadID, resolvedBy, note string, now time.Time) (*t.KafkaDeadLetter, error) {
tenantAdp, err := a.infrastructureAdapter(id)
if err != nil {
return nil, err
}
deadID = strings.TrimSpace(deadID)
if deadID == "" {
return nil, t.ErrMalformed
}
if tenantAdp != a {
return tenantAdp.TenantKafkaDeadLetterMarkReplay(id, deadID, resolvedBy, note, now)
}
tx, err := a.db.BeginTxx(context.Background(), nil)
if err != nil {
return nil, err
}
defer tx.Rollback()
var row struct {
EventID string `db:"event_id"`
ConsumerGroup string `db:"consumer_group"`
}
if err = tx.Get(&row, `SELECT event_id,consumer_group FROM im_kafka_dead_letters WHERE dead_id=? FOR UPDATE`, deadID); err != nil {
if err == sql.ErrNoRows {
return nil, t.ErrNotFound
}
return nil, err
}
res, err := tx.Exec(`UPDATE im_kafka_dead_letters SET status=?,replay_count=replay_count+1,resolved_by=?,
resolved_at=?,resolve_note=?,updated_at=? WHERE dead_id=? AND status IN (?,?)`,
t.KafkaDeadStatusPendingReplay, truncateKafkaString(resolvedBy, 64), now, truncateKafkaString(note, 1000),
now, deadID, t.KafkaDeadStatusDead, t.KafkaDeadStatusPendingReplay)
if err != nil {
return nil, err
}
if affected, _ := res.RowsAffected(); affected == 0 {
return nil, t.ErrNotFound
}
if _, err = tx.Exec(`DELETE FROM im_kafka_consume_log WHERE event_id=? AND consumer_group=?`,
row.EventID, row.ConsumerGroup); err != nil {
return nil, err
}
if err = tx.Commit(); err != nil {
return nil, err
}
return a.TenantKafkaDeadLetterGet(id, deadID)
}
func (a *adapter) TenantKafkaOutboxCreate(id t.TenantID, record *t.KafkaOutboxRecord) error {
tenantAdp, err := a.infrastructureAdapter(id)
if err != nil {
return err
}
if record == nil || record.EventID == "" || record.Topic == "" || record.EventType == "" || len(record.Envelope) == 0 {
return t.ErrMalformed
}
if tenantAdp != a {
return tenantAdp.TenantKafkaOutboxCreate(id, record)
}
now := t.TimeNow()
_, err = a.db.Exec(`INSERT INTO im_kafka_outbox
(event_id,topic,event_type,biz_type,biz_id,key_value,envelope,status,next_retry_at,created_at,updated_at)
VALUES(?,?,?,?,?,?,?,?,?,?,?)
ON DUPLICATE KEY UPDATE updated_at=updated_at`,
record.EventID, record.Topic, record.EventType, record.BizType, record.BizID, record.Key,
record.Envelope, t.KafkaOutboxPending, record.NextRetryAt, now, now)
return err
}
func (a *adapter) TenantKafkaOutboxListPending(id t.TenantID, limit int, now time.Time) ([]t.KafkaOutboxRecord, error) {
tenantAdp, err := a.infrastructureAdapter(id)
if err != nil {
return nil, err
}
if limit <= 0 {
limit = 100
}
if tenantAdp != a {
return tenantAdp.TenantKafkaOutboxListPending(id, limit, now)
}
rows := []struct {
ID int64 `db:"id"`
EventID string `db:"event_id"`
Topic string `db:"topic"`
EventType string `db:"event_type"`
BizType string `db:"biz_type"`
BizID string `db:"biz_id"`
Key string `db:"key_value"`
Envelope []byte `db:"envelope"`
Status string `db:"status"`
Attempts int `db:"attempts"`
NextRetryAt sql.NullTime `db:"next_retry_at"`
SentAt sql.NullTime `db:"sent_at"`
LastReason string `db:"last_reason"`
LastError string `db:"last_error"`
CreatedAt time.Time `db:"created_at"`
UpdatedAt time.Time `db:"updated_at"`
}{}
err = a.db.Select(&rows, `SELECT id,event_id,topic,event_type,biz_type,biz_id,key_value,envelope,status,attempts,
next_retry_at,sent_at,last_reason,last_error,created_at,updated_at
FROM im_kafka_outbox
WHERE status IN (?,?) AND (next_retry_at IS NULL OR next_retry_at<=?)
ORDER BY id ASC LIMIT ?`, t.KafkaOutboxPending, t.KafkaOutboxFailed, now, limit)
if err != nil {
return nil, err
}
out := make([]t.KafkaOutboxRecord, 0, len(rows))
for _, row := range rows {
rec := t.KafkaOutboxRecord{
ID: row.ID,
EventID: row.EventID,
Topic: row.Topic,
EventType: row.EventType,
BizType: row.BizType,
BizID: row.BizID,
Key: row.Key,
Envelope: row.Envelope,
Status: row.Status,
Attempts: row.Attempts,
LastReason: row.LastReason,
LastError: row.LastError,
CreatedAt: row.CreatedAt,
UpdatedAt: row.UpdatedAt,
}
if row.NextRetryAt.Valid {
rec.NextRetryAt = &row.NextRetryAt.Time
}
if row.SentAt.Valid {
rec.SentAt = &row.SentAt.Time
}
out = append(out, rec)
}
return out, nil
}
func (a *adapter) TenantKafkaOutboxMarkSent(id t.TenantID, eventID string, now time.Time) error {
tenantAdp, err := a.infrastructureAdapter(id)
if err != nil {
return err
}
if eventID == "" {
return t.ErrMalformed
}
if tenantAdp != a {
return tenantAdp.TenantKafkaOutboxMarkSent(id, eventID, now)
}
res, err := a.db.Exec(`UPDATE im_kafka_outbox SET status=?,sent_at=?,attempts=attempts+1,last_reason='',
last_error='',updated_at=? WHERE event_id=? AND status<>?`,
t.KafkaOutboxSent, now, now, eventID, t.KafkaOutboxSent)
if err != nil {
return err
}
if affected, _ := res.RowsAffected(); affected == 0 {
return t.ErrNotFound
}
return nil
}
func (a *adapter) TenantKafkaOutboxMarkFailed(id t.TenantID, eventID, reason, message string, nextRetryAt, now time.Time) error {
tenantAdp, err := a.infrastructureAdapter(id)
if err != nil {
return err
}
if eventID == "" {
return t.ErrMalformed
}
if tenantAdp != a {
return tenantAdp.TenantKafkaOutboxMarkFailed(id, eventID, reason, message, nextRetryAt, now)
}
res, err := a.db.Exec(`UPDATE im_kafka_outbox SET status=?,attempts=attempts+1,next_retry_at=?,
last_reason=?,last_error=?,updated_at=? WHERE event_id=? AND status<>?`,
t.KafkaOutboxFailed, nextRetryAt, truncateKafkaString(reason, 64), truncateKafkaString(message, 1000),
now, eventID, t.KafkaOutboxSent)
if err != nil {
return err
}
if affected, _ := res.RowsAffected(); affected == 0 {
return t.ErrNotFound
}
return nil
}
func scanKafkaDeadLetter(rows *sqlx.Rows) (*t.KafkaDeadLetter, error) {
var row struct {
ID int64 `db:"id"`
DeadID string `db:"dead_id"`
EventID string `db:"event_id"`
ParentEventID string `db:"parent_event_id"`
SourceTopic string `db:"source_topic"`
ConsumerGroup string `db:"consumer_group"`
EventType string `db:"event_type"`
BizType string `db:"biz_type"`
BizID string `db:"biz_id"`
Envelope []byte `db:"envelope"`
RetryCount int `db:"retry_count"`
LastReason string `db:"last_reason"`
LastError string `db:"last_error"`
Status string `db:"status"`
ReplayCount int `db:"replay_count"`
CreatedAt time.Time `db:"created_at"`
UpdatedAt time.Time `db:"updated_at"`
}
if err := rows.StructScan(&row); err != nil {
return nil, err
}
return &t.KafkaDeadLetter{
ID: row.ID,
DeadID: row.DeadID,
EventID: row.EventID,
ParentEventID: row.ParentEventID,
SourceTopic: row.SourceTopic,
ConsumerGroup: row.ConsumerGroup,
EventType: row.EventType,
BizType: row.BizType,
BizID: row.BizID,
Envelope: row.Envelope,
RetryCount: row.RetryCount,
LastReason: row.LastReason,
LastError: row.LastError,
Status: row.Status,
ReplayCount: row.ReplayCount,
CreatedAt: row.CreatedAt,
UpdatedAt: row.UpdatedAt,
}, nil
}
func (a *adapter) kafkaConsumeGet(eventID, consumerGroup string) (*t.KafkaConsumeRecord, error) {
row := struct {
EventID string `db:"event_id"`
ConsumerGroup string `db:"consumer_group"`
SourceTopic string `db:"source_topic"`
EventType string `db:"event_type"`
BizType string `db:"biz_type"`
BizID string `db:"biz_id"`
Status string `db:"status"`
RetryCount int `db:"retry_count"`
LastReason string `db:"last_reason"`
LastError string `db:"last_error"`
ProcessingDeadline *time.Time `db:"processing_deadline"`
SucceededAt *time.Time `db:"succeeded_at"`
DeadAt *time.Time `db:"dead_at"`
CreatedAt time.Time `db:"created_at"`
UpdatedAt time.Time `db:"updated_at"`
}{}
err := a.db.Get(&row, `SELECT event_id,consumer_group,source_topic,event_type,biz_type,biz_id,status,retry_count,
last_reason,last_error,processing_deadline,succeeded_at,dead_at,created_at,updated_at
FROM im_kafka_consume_log WHERE event_id=? AND consumer_group=?`, eventID, consumerGroup)
if err == sql.ErrNoRows {
return nil, t.ErrNotFound
}
if err != nil {
return nil, err
}
return &t.KafkaConsumeRecord{EventID: row.EventID, ConsumerGroup: row.ConsumerGroup, SourceTopic: row.SourceTopic,
EventType: row.EventType, BizType: row.BizType, BizID: row.BizID, Status: row.Status, RetryCount: row.RetryCount,
LastReason: row.LastReason, LastError: row.LastError, ProcessingDeadline: row.ProcessingDeadline,
SucceededAt: row.SucceededAt, DeadAt: row.DeadAt, CreatedAt: row.CreatedAt, UpdatedAt: row.UpdatedAt}, nil
}
func truncateKafkaString(value string, limit int) string {
if limit <= 0 {
return ""
}
runes := []rune(value)
if len(runes) <= limit {
return value
}
return string(runes[:limit])
}
func insertBusinessEventTx(tx *sqlx.Tx, eventID, targetType, target string, actor t.Uid, eventType string, payload any) error {
encoded, err := json.Marshal(payload)
if err != nil {
@@ -0,0 +1,126 @@
//go:build mysql
// +build mysql
package mysql
import (
"database/sql"
"encoding/json"
"strings"
"time"
"github.com/jmoiron/sqlx"
t "github.com/tinode/chat/server/store/types"
)
func (a *adapter) TenantInternalAdminQuery(id t.TenantID, query string, countQuery string, args []any, page, perPage int) (*t.InternalAdminQueryResult, error) {
tenantAdp, err := a.routedTenantAdapter(id)
if err != nil {
return nil, t.ErrMalformed
}
if tenantAdp != a {
return tenantAdp.TenantInternalAdminQuery(id, query, countQuery, args, page, perPage)
}
if strings.TrimSpace(query) == "" || strings.TrimSpace(countQuery) == "" {
return nil, t.ErrMalformed
}
if page <= 0 {
page = 1
}
if perPage <= 0 || perPage > 200 {
perPage = 20
}
offset := (page - 1) * perPage
ctx, cancel := a.getContext()
if cancel != nil {
defer cancel()
}
var total int64
if err = a.db.GetContext(ctx, &total, countQuery, args...); err != nil {
return nil, err
}
queryArgs := append(append([]any{}, args...), perPage, offset)
rows, err := a.db.QueryxContext(ctx, query+" LIMIT ? OFFSET ?", queryArgs...)
if err != nil {
return nil, err
}
defer rows.Close()
items, err := scanInternalAdminRows(rows)
if err != nil {
return nil, err
}
return &t.InternalAdminQueryResult{Items: items, Total: total, Page: page, PerPage: perPage}, nil
}
func (a *adapter) TenantInternalAdminQueryOne(id t.TenantID, query string, args []any) (map[string]any, error) {
tenantAdp, err := a.routedTenantAdapter(id)
if err != nil {
return nil, t.ErrMalformed
}
if tenantAdp != a {
return tenantAdp.TenantInternalAdminQueryOne(id, query, args)
}
if strings.TrimSpace(query) == "" {
return nil, t.ErrMalformed
}
ctx, cancel := a.getContext()
if cancel != nil {
defer cancel()
}
rows, err := a.db.QueryxContext(ctx, query+" LIMIT 1", args...)
if err != nil {
return nil, err
}
defer rows.Close()
items, err := scanInternalAdminRows(rows)
if err != nil {
return nil, err
}
if len(items) == 0 {
return nil, t.ErrNotFound
}
return items[0], nil
}
func scanInternalAdminRows(rows *sqlx.Rows) ([]map[string]any, error) {
items := []map[string]any{}
for rows.Next() {
row := map[string]any{}
if err := rows.MapScan(row); err != nil {
return nil, err
}
for key, value := range row {
row[key] = normalizeInternalAdminValue(value)
}
items = append(items, row)
}
return items, rows.Err()
}
func normalizeInternalAdminValue(value any) any {
switch v := value.(type) {
case nil:
return nil
case []byte:
raw := strings.TrimSpace(string(v))
if raw == "" {
return ""
}
if (strings.HasPrefix(raw, "{") && strings.HasSuffix(raw, "}")) || (strings.HasPrefix(raw, "[") && strings.HasSuffix(raw, "]")) {
var parsed any
if json.Unmarshal([]byte(raw), &parsed) == nil {
return parsed
}
}
return raw
case time.Time:
return v.UTC().Format(time.RFC3339Nano)
case sql.NullTime:
if v.Valid {
return v.Time.UTC().Format(time.RFC3339Nano)
}
return nil
default:
return value
}
}
+13
View File
@@ -10,6 +10,7 @@
package main
import (
"context"
"encoding/base64"
"encoding/json"
"errors"
@@ -349,6 +350,9 @@ func largeFileReceiveHTTP(wrt http.ResponseWriter, req *http.Request) {
writeHttpResponse(decodeStoreError(err, msgID, now, nil), err)
return
}
if err = enqueueKafkaFileUploaded(req.Context(), principal.TenantID, fdef); err != nil {
logs.Warn.Printf("media upload: failed to enqueue file lifecycle event file=%s: %v", fdef.Id, err)
}
params := map[string]string{"url": url}
if globals.mediaGcPeriod > 0 {
@@ -628,6 +632,15 @@ func (*grpcNodeServer) LargeFileReceive(stream pbx.Node_LargeFileReceiveServer)
Size: size,
},
})
if err == nil {
if fdef, finishErr := store.Files.FinishUpload(principal.TenantID, fdef, true, size); finishErr == nil {
if lifecycleErr := enqueueKafkaFileUploaded(context.Background(), principal.TenantID, fdef); lifecycleErr != nil {
logs.Warn.Printf("media upload: failed to enqueue grpc file lifecycle event file=%s: %v", fdef.Id, lifecycleErr)
}
} else {
err = finishErr
}
}
logs.Info.Println("media upload: ok", fdef.Id, fdef.Location, err)
return err
}
+243 -14
View File
@@ -68,7 +68,7 @@ const (
imQRCodeMaxExpiresInSeconds = 7 * 86400
imPublicUploadMaxBytes = 10 << 20
groupMaxAdminCount = 10
monthlyReportPreviewMaxRunes = 120
reportPreviewMaxRunes = 120
)
type imHTTPResponse struct {
@@ -170,6 +170,10 @@ func serveIMHTTPIdempotent(wrt http.ResponseWriter, req *http.Request, next http
next.ServeHTTP(wrt, req)
return
}
if req.Method == http.MethodPost && strings.TrimSpace(req.URL.Path) == "/v1/im/tenants/resolve" {
next.ServeHTTP(wrt, req)
return
}
if !strings.HasPrefix(strings.TrimSpace(req.Header.Get("Authorization")), "Bearer ") {
next.ServeHTTP(wrt, req)
return
@@ -396,6 +400,16 @@ type imFavoritesForwardRequest struct {
Comment string `json:"comment,omitempty"`
}
type imMessagesForwardRequest struct {
Mode string `json:"mode"`
Sources []struct {
Topic string `json:"topic"`
Seq int64 `json:"seq,string"`
} `json:"sources"`
TargetTopics []string `json:"target_topics"`
Comment string `json:"comment,omitempty"`
}
type imFriendRequestCreateRequest struct {
TargetUserID string `json:"target_user_id"`
Message string `json:"message,omitempty"`
@@ -1886,6 +1900,8 @@ func serveIMMeetings(wrt http.ResponseWriter, req *http.Request) {
return
}
}
publishIMWorkbenchAndAudit(req.Context(), principal.TenantID, principal.Uid, "meeting", "meeting.created", meeting.MeetingID,
meetingParticipantUIDs(meeting), "会议通知", meeting.Title, map[string]any{"meeting_id": meeting.MeetingID, "version": meeting.Version, "state": meeting.State})
writeIMSuccess(wrt, req, meetingPayload(meeting, principal.Uid))
}
@@ -1979,6 +1995,8 @@ func serveIMMeetingUpdate(wrt http.ResponseWriter, req *http.Request, meetingID
return
}
}
publishIMWorkbenchAndAudit(req.Context(), principal.TenantID, principal.Uid, "meeting", "meeting.updated", meeting.MeetingID,
meetingParticipantUIDs(meeting), "会议通知", meeting.Title, map[string]any{"meeting_id": meeting.MeetingID, "version": meeting.Version, "state": meeting.State})
writeIMSuccess(wrt, req, meetingPayload(meeting, principal.Uid))
}
@@ -2006,6 +2024,8 @@ func serveIMMeetingCancel(wrt http.ResponseWriter, req *http.Request, meetingID
return
}
}
publishIMWorkbenchAndAudit(req.Context(), principal.TenantID, principal.Uid, "meeting", "meeting.canceled", meeting.MeetingID,
meetingParticipantUIDs(meeting), "会议通知", meeting.Title, map[string]any{"meeting_id": meeting.MeetingID, "version": meeting.Version, "state": meeting.State})
writeIMSuccess(wrt, req, meetingPayload(meeting, principal.Uid))
}
@@ -2039,6 +2059,8 @@ func serveIMMeetingReminders(wrt http.ResponseWriter, req *http.Request, meeting
return
}
}
publishIMWorkbenchAndAudit(req.Context(), principal.TenantID, principal.Uid, "meeting", "meeting.reminded", meetingID,
meetingParticipantUserIDs(participants), "会议提醒", meetingID, map[string]any{"meeting_id": meetingID, "reminded_count": len(participants)})
writeIMSuccess(wrt, req, map[string]any{
"meeting_id": meetingID,
"reminded": meetingParticipantPayloads(participants),
@@ -2072,6 +2094,8 @@ func serveIMMeetingMyResponse(wrt http.ResponseWriter, req *http.Request, meetin
return
}
}
publishIMWorkbenchAndAudit(req.Context(), principal.TenantID, principal.Uid, "meeting", "meeting.response.updated", meeting.MeetingID,
meetingParticipantUIDs(meeting), "会议通知", meeting.Title, map[string]any{"meeting_id": meeting.MeetingID, "version": meeting.Version, "state": meeting.State})
writeIMSuccess(wrt, req, meetingPayload(meeting, principal.Uid))
}
@@ -2126,6 +2150,8 @@ func serveIMDailyReports(wrt http.ResponseWriter, req *http.Request) {
writeIMRelationError(wrt, req, err)
return
}
publishIMWorkbenchAndAudit(req.Context(), principal.TenantID, principal.Uid, "daily_report", "daily_report.draft.created", report.ReportID,
[]types.Uid{principal.Uid}, "日报", report.ReportDate.Format("2006-01-02"), map[string]any{"report_id": report.ReportID, "version": report.Version, "status": report.Status})
writeIMSuccess(wrt, req, dailyReportDetailPayload(report, principal.Uid))
default:
writeIMError(wrt, req, http.StatusMethodNotAllowed, 30002, imReasonMethodNotAllowed, "method not allowed", nil)
@@ -2172,6 +2198,8 @@ func serveIMDailyReportGet(wrt http.ResponseWriter, req *http.Request, reportID
writeIMRelationError(wrt, req, err)
return
}
publishIMWorkbenchAndAudit(req.Context(), principal.TenantID, principal.Uid, "daily_report", "daily_report.draft.saved", report.ReportID,
[]types.Uid{principal.Uid}, "日报", report.ReportDate.Format("2006-01-02"), map[string]any{"report_id": report.ReportID, "version": report.Version, "status": report.Status})
writeIMSuccess(wrt, req, dailyReportDetailPayload(report, principal.Uid))
}
@@ -2197,6 +2225,8 @@ func serveIMDailyReportSaveDraft(wrt http.ResponseWriter, req *http.Request, rep
writeIMRelationError(wrt, req, err)
return
}
publishIMWorkbenchAndAudit(req.Context(), principal.TenantID, principal.Uid, "daily_report", "daily_report.submitted", report.ReportID,
[]types.Uid{principal.Uid}, "日报已提交", report.ReportDate.Format("2006-01-02"), map[string]any{"report_id": report.ReportID, "version": report.Version, "status": report.Status})
writeIMSuccess(wrt, req, dailyReportDetailPayload(report, principal.Uid))
}
@@ -2288,6 +2318,8 @@ func serveIMDailyReportReminders(wrt http.ResponseWriter, req *http.Request) {
for _, uid := range reminded {
items = append(items, map[string]any{"user_id": uid.UserId(), "status": "queued"})
}
publishIMWorkbenchAndAudit(req.Context(), principal.TenantID, principal.Uid, "daily_report", "daily_report.reminded", date.Format("2006-01-02"),
reminded, "日报提醒", date.Format("2006-01-02"), map[string]any{"date": date.Format("2006-01-02"), "reminded_count": len(reminded)})
writeIMSuccess(wrt, req, map[string]any{"date": date.Format("2006-01-02"), "items": items})
}
@@ -2332,6 +2364,8 @@ func serveIMMonthlyReports(wrt http.ResponseWriter, req *http.Request) {
writeIMRelationError(wrt, req, err)
return
}
publishIMWorkbenchAndAudit(req.Context(), principal.TenantID, principal.Uid, "monthly_report", "monthly_report.draft.created", report.ReportID,
[]types.Uid{principal.Uid}, "月报", report.Month, map[string]any{"report_id": report.ReportID, "version": report.Version, "status": report.Status})
writeIMSuccess(wrt, req, monthlyReportDetailPayload(report, principal.Uid))
default:
writeIMError(wrt, req, http.StatusMethodNotAllowed, 30002, imReasonMethodNotAllowed, "method not allowed", nil)
@@ -2377,6 +2411,8 @@ func serveIMMonthlyReportGet(wrt http.ResponseWriter, req *http.Request, reportI
writeIMRelationError(wrt, req, err)
return
}
publishIMWorkbenchAndAudit(req.Context(), principal.TenantID, principal.Uid, "monthly_report", "monthly_report.draft.saved", report.ReportID,
[]types.Uid{principal.Uid}, "月报", report.Month, map[string]any{"report_id": report.ReportID, "version": report.Version, "status": report.Status})
writeIMSuccess(wrt, req, monthlyReportDetailPayload(report, principal.Uid))
}
@@ -2403,6 +2439,8 @@ func serveIMMonthlyReportSaveDraft(wrt http.ResponseWriter, req *http.Request, r
writeIMRelationError(wrt, req, err)
return
}
publishIMWorkbenchAndAudit(req.Context(), principal.TenantID, principal.Uid, "monthly_report", "monthly_report.submitted", report.ReportID,
[]types.Uid{principal.Uid}, "月报已提交", report.Month, map[string]any{"report_id": report.ReportID, "version": report.Version, "status": report.Status})
writeIMSuccess(wrt, req, monthlyReportDetailPayload(report, principal.Uid))
}
@@ -2426,6 +2464,8 @@ func serveIMMonthlyReportSubmit(wrt http.ResponseWriter, req *http.Request, repo
writeIMRelationError(wrt, req, err)
return
}
publishIMWorkbenchAndAudit(req.Context(), principal.TenantID, principal.Uid, "monthly_report", "monthly_report.reopened", report.ReportID,
[]types.Uid{principal.Uid}, "月报已重开", report.Month, map[string]any{"report_id": report.ReportID, "version": report.Version, "status": report.Status})
writeIMSuccess(wrt, req, monthlyReportDetailPayload(report, principal.Uid))
}
@@ -2541,6 +2581,8 @@ func serveIMMailboxSubmissions(wrt http.ResponseWriter, req *http.Request) {
writeIMRelationError(wrt, req, err)
return
}
publishIMWorkbenchAndAudit(req.Context(), principal.TenantID, principal.Uid, "mailbox", "mailbox.submitted", submission.SubmissionID,
[]types.Uid{principal.Uid}, "总经理信箱", submission.Subject, map[string]any{"submission_id": submission.SubmissionID, "status": submission.Status, "anonymous": submission.Anonymous})
writeIMSuccess(wrt, req, mailboxSubmissionResultPayload(submission))
}
@@ -3201,7 +3243,7 @@ func serveIMFavoriteForward(wrt http.ResponseWriter, req *http.Request) {
},
Content: favoriteForwardContent(item, comment),
}
saved, err := store.Messages.Save(principal.TenantID, msg, nil, true)
saved, err := saveMessageMaybeKafka(req.Context(), principal.TenantID, msg, nil, true)
if err != nil {
results = append(results, map[string]any{"target_topic": target, "favorite_id": favoriteID, "status": "failed", "reason": relationErrorReason(err)})
continue
@@ -3212,6 +3254,107 @@ func serveIMFavoriteForward(wrt http.ResponseWriter, req *http.Request) {
writeIMSuccess(wrt, req, map[string]any{"items": results})
}
func serveIMMessageForward(wrt http.ResponseWriter, req *http.Request) {
if req.Method != http.MethodPost {
writeIMError(wrt, req, http.StatusMethodNotAllowed, 30002, imReasonMethodNotAllowed, "method not allowed", nil)
return
}
principal, ok := authenticateIMHTTPBearer(wrt, req)
if !ok {
return
}
var body imMessagesForwardRequest
if !decodeIMJSONBody(wrt, req, &body) {
return
}
mode := strings.TrimSpace(body.Mode)
if mode == "" {
mode = "separate"
}
if mode != "separate" {
writeIMError(wrt, req, http.StatusNotImplemented, 30004, "NOT_IMPLEMENTED", "forward bundle is not implemented", map[string]any{"field": "mode"})
return
}
if len(body.Sources) == 0 || len(body.Sources) > 100 || len(body.TargetTopics) == 0 || len(body.TargetTopics) > 20 {
writeIMError(wrt, req, http.StatusBadRequest, 30000, imReasonMalformedRequest, "malformed request", nil)
return
}
sourceReadable := make(map[string]error)
targetWritable := make(map[string]error)
items := make([]map[string]any, 0, len(body.TargetTopics)*len(body.Sources))
failed := make([]map[string]any, 0)
for _, rawTarget := range body.TargetTopics {
target := strings.TrimSpace(rawTarget)
if target == "" {
for _, source := range body.Sources {
failed = append(failed, forwardFailurePayload(source.Topic, source.Seq, target, imReasonMalformedRequest))
}
continue
}
targetErr, ok := targetWritable[target]
if !ok {
targetErr = requireTopicWriter(principal, target)
targetWritable[target] = targetErr
}
if targetErr != nil {
for _, source := range body.Sources {
failed = append(failed, forwardFailurePayload(source.Topic, source.Seq, target, relationErrorReason(targetErr)))
}
continue
}
for _, source := range body.Sources {
sourceTopic := strings.TrimSpace(source.Topic)
if sourceTopic == "" || source.Seq <= 0 {
failed = append(failed, forwardFailurePayload(sourceTopic, source.Seq, target, imReasonMalformedRequest))
continue
}
readErr, ok := sourceReadable[sourceTopic]
if !ok {
readErr = requireTopicReader(principal, sourceTopic)
sourceReadable[sourceTopic] = readErr
}
if readErr != nil {
failed = append(failed, forwardFailurePayload(sourceTopic, source.Seq, target, relationErrorReason(readErr)))
continue
}
msg, err := store.Messages.GetOne(principal.TenantID, sourceTopic, source.Seq)
if err != nil {
failed = append(failed, forwardFailurePayload(sourceTopic, source.Seq, target, relationErrorReason(err)))
continue
}
if msg == nil {
failed = append(failed, forwardFailurePayload(sourceTopic, source.Seq, target, "NOT_FOUND"))
continue
}
if messageAvailability(*msg) != "available" {
failed = append(failed, forwardFailurePayload(sourceTopic, source.Seq, target, "MESSAGE_UNAVAILABLE"))
continue
}
clientMid := "fwd_" + store.Store.GetUidString()
forwarded := &types.Message{
TenantID: principal.TenantID,
Topic: target,
From: principal.Uid.String(),
Head: forwardedMessageHead(msg, clientMid),
Content: forwardedMessageContent(msg, body.Comment),
}
saved, err := saveMessageMaybeKafka(req.Context(), principal.TenantID, forwarded, nil, true)
if err != nil {
failed = append(failed, forwardFailurePayload(sourceTopic, source.Seq, target, relationErrorReason(err)))
continue
}
items = append(items, map[string]any{
"source": map[string]any{"topic": sourceTopic, "seq": source.Seq},
"target_topic": target,
"client_mid": clientMid,
"seq": saved.SeqId,
"status": "sent",
})
}
}
writeIMSuccess(wrt, req, map[string]any{"items": items, "failed": failed})
}
func serveIMTopicMessageAction(wrt http.ResponseWriter, req *http.Request) {
switch {
case strings.HasSuffix(req.URL.Path, "/draft"):
@@ -4500,6 +4643,9 @@ func serveIMGroupFiles(wrt http.ResponseWriter, req *http.Request, topic, tail s
writeIMRelationError(wrt, req, err)
return
}
if err = enqueueKafkaGroupFileLifecycle(req.Context(), principal.TenantID, kafkaFileLifecycleGroupFileCreate, file, principal.Uid); err != nil {
logs.Warn.Printf("im http: failed to enqueue group file lifecycle event topic=%s file=%s: %v", topic, file.FileID, err)
}
writeIMSuccess(wrt, req, groupFilePayload(file))
case strings.HasPrefix(tail, "files/") && req.Method == http.MethodDelete:
groupFileID := strings.TrimSpace(strings.TrimPrefix(tail, "files/"))
@@ -4513,6 +4659,9 @@ func serveIMGroupFiles(wrt http.ResponseWriter, req *http.Request, topic, tail s
writeIMRelationError(wrt, req, err)
return
}
if err = enqueueKafkaGroupFileLifecycle(req.Context(), principal.TenantID, kafkaFileLifecycleGroupFileDelete, file, principal.Uid); err != nil {
logs.Warn.Printf("im http: failed to enqueue group file lifecycle event topic=%s file=%s: %v", topic, file.FileID, err)
}
writeIMSuccess(wrt, req, groupFilePayload(file))
default:
writeIMError(wrt, req, http.StatusMethodNotAllowed, 30002, imReasonMethodNotAllowed, "method not allowed", nil)
@@ -7134,6 +7283,33 @@ func meetingParticipantPayloads(participants []types.MeetingParticipant) []map[s
return items
}
func meetingParticipantUIDs(meeting *types.Meeting) []types.Uid {
if meeting == nil {
return nil
}
return meetingParticipantUserIDs(meeting.Participants)
}
func meetingParticipantUserIDs(participants []types.MeetingParticipant) []types.Uid {
items := make([]types.Uid, 0, len(participants))
for _, participant := range participants {
if !participant.UserID.IsZero() {
items = append(items, participant.UserID)
}
}
return items
}
func publishIMWorkbenchAndAudit(ctx context.Context, tenantID types.TenantID, actor types.Uid, module, action, resourceID string,
targets []types.Uid, title, summary string, data map[string]any) {
if err := enqueueKafkaWorkbenchEvent(ctx, tenantID, actor, module, action, resourceID, targets, title, summary, data); err != nil {
logs.Warn.Printf("im http: failed to enqueue workbench event action=%s resource=%s: %v", action, resourceID, err)
}
if err := enqueueKafkaAuditEvent(ctx, tenantID, "user", actor, action, module, resourceID, "succeeded", "", data); err != nil {
logs.Warn.Printf("im http: failed to enqueue audit event action=%s resource=%s: %v", action, resourceID, err)
}
}
func meetingPermissionsPayload(meeting *types.Meeting, viewer types.Uid) map[string]any {
isOrganizer := meeting.OrganizerID == viewer
isScheduled := meeting.State == types.MeetingStateScheduled
@@ -7168,7 +7344,7 @@ func dailyReportListPayload(report types.DailyReport) map[string]any {
"report_id": report.ReportID,
"author_id": report.AuthorID.UserId(),
"report_date": report.ReportDate.Format("2006-01-02"),
"content_short": monthlyReportPreview(report.Content),
"content_short": reportPreview(report.Content),
"status": report.Status,
"draft_updated_at": report.UpdatedAt.UTC().Format(time.RFC3339Nano),
"read_state": report.ReadState,
@@ -7211,8 +7387,8 @@ func monthlyReportListPayload(report types.MonthlyReport) map[string]any {
"report_id": report.ReportID,
"author_id": report.AuthorID.UserId(),
"month": report.Month,
"work_summary_short": monthlyReportPreview(report.WorkSummary),
"next_plan_short": monthlyReportPreview(report.NextPlan),
"work_summary_short": reportPreview(report.WorkSummary),
"next_plan_short": reportPreview(report.NextPlan),
"status": report.Status,
"read_state": report.ReadState,
"version": report.Version,
@@ -7225,13 +7401,13 @@ func monthlyReportListPayload(report types.MonthlyReport) map[string]any {
return data
}
func monthlyReportPreview(text string) string {
func reportPreview(text string) string {
text = strings.Join(strings.Fields(text), " ")
runes := []rune(text)
if len(runes) <= monthlyReportPreviewMaxRunes {
if len(runes) <= reportPreviewMaxRunes {
return text
}
return string(runes[:monthlyReportPreviewMaxRunes]) + "..."
return string(runes[:reportPreviewMaxRunes]) + "..."
}
func monthlyReportDetailPayload(report *types.MonthlyReport, viewer types.Uid) map[string]any {
@@ -7826,6 +8002,20 @@ func requireTopicWriter(principal *imPrincipal, topic string) error {
return nil
}
func requireTopicReader(principal *imPrincipal, topic string) error {
if principal == nil || topic == "" {
return types.ErrMalformed
}
sub, err := store.Subs.Get(principal.TenantID, topic, principal.Uid, false)
if err != nil {
return err
}
if sub == nil || !(sub.ModeGiven & sub.ModeWant).IsReader() {
return types.ErrPermissionDenied
}
return nil
}
func copyStringAnyMap(in map[string]any) map[string]any {
out := make(map[string]any, len(in))
for key, value := range in {
@@ -7956,6 +8146,49 @@ func favoriteForwardContent(item *types.FavoriteItem, comment string) map[string
return payload
}
func forwardFailurePayload(topic string, seq int64, target, reason string) map[string]any {
return map[string]any{
"source": map[string]any{"topic": strings.TrimSpace(topic), "seq": seq},
"target_topic": strings.TrimSpace(target),
"status": "failed",
"reason": reason,
}
}
func forwardedMessageHead(msg *types.Message, clientMid string) types.KVMap {
head := types.KVMap{}
if msg != nil {
for key, value := range msg.Head {
head[key] = value
}
msgType := imMessageType(msg)
if strings.TrimSpace(fmt.Sprint(head["type"])) == "" || fmt.Sprint(head["type"]) == "<nil>" {
head["type"] = msgType
}
if strings.TrimSpace(fmt.Sprint(head["x-im-type"])) == "" || fmt.Sprint(head["x-im-type"]) == "<nil>" {
head["x-im-type"] = msgType
}
head["x-im-forward-source"] = map[string]any{"topic": msg.Topic, "seq": msg.SeqId, "from": msg.From}
}
head["x-im-forwarded"] = true
head["x-im-client-mid"] = clientMid
return head
}
func forwardedMessageContent(msg *types.Message, comment string) any {
if strings.TrimSpace(comment) == "" {
if msg == nil {
return nil
}
return msg.Content
}
return map[string]any{
"forwarded": true,
"comment": strings.TrimSpace(comment),
"snapshot": favoriteSnapshot(msg),
}
}
func classifyIMURLRisk(rawURL string) (decision, riskLevel, reason string) {
if rawURL == "" || len(rawURL) > 2048 {
return "block", "high", "invalid_url"
@@ -8802,14 +9035,10 @@ func peerFromP2PTopic(topic string, uid types.Uid) types.Uid {
if types.GetTopicCat(topic) != types.TopicCatP2P || uid.IsZero() {
return types.ZeroUid
}
prefix := "p2p"
body := strings.TrimPrefix(topic, prefix)
uidLen := len(uid.String())
if len(body) < 2*uidLen {
left, right, err := types.ParseP2P(topic)
if err != nil {
return types.ZeroUid
}
left := types.ParseUid(body[:uidLen])
right := types.ParseUid(body[uidLen:])
if left == uid {
return right
}
+194 -36
View File
@@ -40,6 +40,7 @@ type testVerificationStore struct {
type testHTTPInfrastructureStore struct {
records map[string]*types.HTTPIdempotencyRecord
events []types.BusinessEvent
outbox []types.KafkaOutboxRecord
}
type testTenantEntryStore struct {
@@ -147,6 +148,57 @@ func (s *testHTTPInfrastructureStore) MarkEventDelivered(_ types.TenantID, _ str
return nil
}
func (s *testHTTPInfrastructureStore) BeginKafkaConsume(_ types.TenantID, record *types.KafkaConsumeRecord, _ time.Time) (*types.KafkaConsumeRecord, bool, error) {
copy := *record
copy.Status = types.KafkaConsumeProcessing
return &copy, true, nil
}
func (s *testHTTPInfrastructureStore) MarkKafkaConsumeSucceeded(_ types.TenantID, _, _ string, _ time.Time) error {
return nil
}
func (s *testHTTPInfrastructureStore) MarkKafkaConsumeFailed(_ types.TenantID, eventID, group, reason, message string, _ time.Time) (*types.KafkaConsumeRecord, error) {
return &types.KafkaConsumeRecord{EventID: eventID, ConsumerGroup: group, Status: types.KafkaConsumeFailed, RetryCount: 1, LastReason: reason, LastError: message}, nil
}
func (s *testHTTPInfrastructureStore) MarkKafkaConsumeDead(_ types.TenantID, _ *types.KafkaDeadLetter, _ time.Time) error {
return nil
}
func (s *testHTTPInfrastructureStore) ListKafkaDeadLetters(_ types.TenantID, _ types.KafkaDeadLetterListOptions) (*types.KafkaDeadLetterPage, error) {
return &types.KafkaDeadLetterPage{}, nil
}
func (s *testHTTPInfrastructureStore) GetKafkaDeadLetter(_ types.TenantID, _ string) (*types.KafkaDeadLetter, error) {
return nil, types.ErrNotFound
}
func (s *testHTTPInfrastructureStore) MarkKafkaDeadLetter(_ types.TenantID, _, _, _, _ string, _ time.Time) error {
return nil
}
func (s *testHTTPInfrastructureStore) MarkKafkaDeadLetterReplay(_ types.TenantID, deadID, _, _ string, _ time.Time) (*types.KafkaDeadLetter, error) {
return &types.KafkaDeadLetter{DeadID: deadID, Status: types.KafkaDeadStatusPendingReplay}, nil
}
func (s *testHTTPInfrastructureStore) CreateKafkaOutbox(_ types.TenantID, record *types.KafkaOutboxRecord) error {
s.outbox = append(s.outbox, *record)
return nil
}
func (s *testHTTPInfrastructureStore) ListPendingKafkaOutbox(_ types.TenantID, _ int, _ time.Time) ([]types.KafkaOutboxRecord, error) {
return append([]types.KafkaOutboxRecord(nil), s.outbox...), nil
}
func (s *testHTTPInfrastructureStore) MarkKafkaOutboxSent(_ types.TenantID, _ string, _ time.Time) error {
return nil
}
func (s *testHTTPInfrastructureStore) MarkKafkaOutboxFailed(_ types.TenantID, _, _, _ string, _, _ time.Time) error {
return nil
}
type testRelationsStore struct {
context *types.RelationshipContext
created *types.FriendRequest
@@ -1261,6 +1313,23 @@ func TestServeIMTenantResolveSuccessFromHeader(t *testing.T) {
}
}
func TestIMHTTPProtocolDoesNotRequireIdempotencyForTenantResolve(t *testing.T) {
next := http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
writeIMSuccess(w, req, map[string]any{"ok": true})
})
req := httptest.NewRequest(http.MethodPost, "/v1/im/tenants/resolve", nil)
req.Header.Set("X-Tinode-APIKey", makeTestAPIKey(t))
req.Header.Set("X-IM-Schema", "1")
req.Header.Set("Authorization", "Bearer test-token")
rec := httptest.NewRecorder()
withIMHTTPProtocol("/v1/im/", next).ServeHTTP(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("status: got %d, want %d; body=%s", rec.Code, http.StatusOK, rec.Body.String())
}
}
func TestServeIMTenantResolveUsesForwardedPublicHost(t *testing.T) {
oldTenants := store.Tenants
oldServingAt := globals.servingAt
@@ -2348,6 +2417,42 @@ func TestServeIMDailyReportsCreateAndList(t *testing.T) {
}
}
func TestServeIMDailyReportsListIncludesContentPreview(t *testing.T) {
ctrl := gomock.NewController(t)
defer ctrl.Finish()
expectIMBearer(t, ctrl, "token-1", 7, 42)
oldDailyReports := store.DailyReports
now := time.Now().UTC().Round(time.Millisecond)
reports := &testDailyReportsStore{report: &types.DailyReport{
ReportID: "dr_test",
AuthorID: 42,
ReportDate: time.Date(2026, 8, 14, 0, 0, 0, 0, time.UTC),
Content: strings.Repeat("日报", 80),
Status: types.DailyReportStatusDraft,
ReadState: "unread",
Version: 1,
CreatedAt: now,
UpdatedAt: now,
}}
store.DailyReports = reports
defer func() { store.DailyReports = oldDailyReports }()
req := httptest.NewRequest(http.MethodGet, "/v1/im/reports/daily?scope=mine", nil)
req.Header.Set("Authorization", "Bearer token-1")
rec := httptest.NewRecorder()
serveIMDailyReports(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("status: got %d, want %d; body=%s", rec.Code, http.StatusOK, rec.Body.String())
}
data := decodeIMHTTPResponse(t, rec).Data.(map[string]any)
items := data["items"].([]any)
item := items[0].(map[string]any)
if item["content_short"] != strings.Repeat("日报", 60)+"..." {
t.Fatalf("unexpected daily report preview: %#v", item["content_short"])
}
}
func TestServeIMDailyReportDraftAndSubmit(t *testing.T) {
ctrl := gomock.NewController(t)
defer ctrl.Finish()
@@ -2419,42 +2524,6 @@ func TestServeIMDailyReportStatisticsAndReminders(t *testing.T) {
}
}
func TestServeIMDailyReportsListIncludesContentPreview(t *testing.T) {
ctrl := gomock.NewController(t)
defer ctrl.Finish()
expectIMBearer(t, ctrl, "token-1", 7, 42)
oldDailyReports := store.DailyReports
now := time.Now().UTC().Round(time.Millisecond)
reports := &testDailyReportsStore{report: &types.DailyReport{
ReportID: "dr_test",
AuthorID: 42,
ReportDate: time.Date(2026, 8, 14, 0, 0, 0, 0, time.UTC),
Content: strings.Repeat("日报", 80),
Status: types.DailyReportStatusDraft,
ReadState: "unread",
Version: 1,
CreatedAt: now,
UpdatedAt: now,
}}
store.DailyReports = reports
defer func() { store.DailyReports = oldDailyReports }()
req := httptest.NewRequest(http.MethodGet, "/v1/im/reports/daily?scope=mine", nil)
req.Header.Set("Authorization", "Bearer token-1")
rec := httptest.NewRecorder()
serveIMDailyReports(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("status: got %d, want %d; body=%s", rec.Code, http.StatusOK, rec.Body.String())
}
data := decodeIMHTTPResponse(t, rec).Data.(map[string]any)
items := data["items"].([]any)
item := items[0].(map[string]any)
if item["content_short"] != strings.Repeat("日报", 60)+"..." {
t.Fatalf("unexpected daily report preview: %#v", item["content_short"])
}
}
func TestServeIMMonthlyReportsCreate(t *testing.T) {
ctrl := gomock.NewController(t)
defer ctrl.Finish()
@@ -3988,6 +4057,83 @@ func TestServeIMFavoriteForwardRejectsUnauthorizedTarget(t *testing.T) {
}
}
func TestServeIMMessageForwardSeparateSavesMessages(t *testing.T) {
ctrl := gomock.NewController(t)
defer ctrl.Finish()
mockStore := expectIMBearer(t, ctrl, "token-1", 7, 42)
mockStore.EXPECT().GetUidString().Return("mid-1")
oldSubs := store.Subs
oldMessages := store.Messages
subs := mock_store.NewMockSubsPersistenceInterface(ctrl)
messages := mock_store.NewMockMessagesPersistenceInterface(ctrl)
store.Subs = subs
store.Messages = messages
t.Cleanup(func() {
store.Subs = oldSubs
store.Messages = oldMessages
})
subs.EXPECT().Get(types.TenantID(7), "grpTarget", types.Uid(42), false).Return(&types.Subscription{
TenantID: 7,
User: types.Uid(42).String(),
Topic: "grpTarget",
ModeWant: types.ModeCPublic,
ModeGiven: types.ModeCPublic,
}, nil)
subs.EXPECT().Get(types.TenantID(7), "grpSource", types.Uid(42), false).Return(&types.Subscription{
TenantID: 7,
User: types.Uid(42).String(),
Topic: "grpSource",
ModeWant: types.ModeCPublic,
ModeGiven: types.ModeCPublic,
}, nil)
messages.EXPECT().GetOne(types.TenantID(7), "grpSource", int64(3)).Return(&types.Message{
TenantID: 7,
Topic: "grpSource",
SeqId: 3,
From: types.Uid(99).String(),
Head: types.KVMap{"x-im-type": "text"},
Content: map[string]any{"text": "hello"},
ObjHeader: types.ObjHeader{
CreatedAt: time.Date(2026, 8, 13, 9, 0, 0, 0, time.UTC),
},
}, nil)
messages.EXPECT().Save(types.TenantID(7), gomock.Any(), gomock.Nil(), true).DoAndReturn(
func(_ types.TenantID, msg *types.Message, _ []string, _ bool) (*types.MessageSaveResult, error) {
if msg.Topic != "grpTarget" || msg.From != types.Uid(42).String() {
t.Fatalf("unexpected forwarded message: %#v", msg)
}
if msg.Head["x-im-forwarded"] != true || msg.Head["x-im-client-mid"] != "fwd_mid-1" {
t.Fatalf("forward metadata not set: %#v", msg.Head)
}
if msg.Content.(map[string]any)["text"] != "hello" {
t.Fatalf("unexpected content: %#v", msg.Content)
}
return &types.MessageSaveResult{SeqId: 21, MarkedReadBySender: true}, nil
})
req := httptest.NewRequest(http.MethodPost, "/v1/im/messages/forward", bytes.NewBufferString(`{"mode":"separate","sources":[{"topic":"grpSource","seq":"3"}],"target_topics":["grpTarget"]}`))
req.Header.Set("Authorization", "Bearer token-1")
rec := httptest.NewRecorder()
serveIMMessageForward(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("forward status: got %d body=%s", rec.Code, rec.Body.String())
}
data := decodeIMHTTPResponse(t, rec).Data.(map[string]any)
items := data["items"].([]any)
if len(items) != 1 {
t.Fatalf("unexpected forward response: %#v", data)
}
got := items[0].(map[string]any)
if got["status"] != "sent" || got["seq"].(float64) != 21 || got["client_mid"] != "fwd_mid-1" {
t.Fatalf("unexpected forward item: %#v", got)
}
if failed := data["failed"].([]any); len(failed) != 0 {
t.Fatalf("unexpected failed items: %#v", failed)
}
}
func TestServeIMSecurityURLCheck(t *testing.T) {
ctrl := gomock.NewController(t)
defer ctrl.Finish()
@@ -4533,6 +4679,18 @@ func TestServeIMConversationContextAcceptsUserTopic(t *testing.T) {
}
}
func TestNormalizeP2PConversationTopicParsesTinodeTopic(t *testing.T) {
user := types.Uid(1)
peer := types.Uid(2)
topic := user.P2PName(peer)
normalized, gotPeer := normalizeP2PConversationTopic(topic, user)
if normalized != topic || gotPeer != peer {
t.Fatalf("normalize p2p: got topic=%q peer=%s, want topic=%q peer=%s", normalized, gotPeer.UserId(), topic, peer.UserId())
}
}
func TestServeIMP2PConversation(t *testing.T) {
ctrl := gomock.NewController(t)
defer ctrl.Finish()
+15
View File
@@ -0,0 +1,15 @@
package main
import "testing"
func TestInternalAdminDBIDKeepsNumericID(t *testing.T) {
const rawID int64 = 1559676214946304000
id, ok := parseInternalAdminDBID("1559676214946304000")
if !ok {
t.Fatal("numeric admin id was rejected")
}
if id != rawID {
t.Fatalf("numeric admin id = %d, want %d", id, rawID)
}
}
+413
View File
@@ -0,0 +1,413 @@
package main
import (
"fmt"
"net/http"
"strconv"
"strings"
"github.com/tinode/chat/server/store"
"github.com/tinode/chat/server/store/types"
)
type internalQueryFilter struct {
Param string
Column string
Op string
Kind string
}
type internalQuerySpec struct {
Select string
From string
DefaultSort string
Filters []internalQueryFilter
DetailKey string
ScopeColumn string
}
var internalGroupResourceSpecs = map[string]*internalQuerySpec{
"join-requests": {
Select: "SELECT id,request_id,topic,applicant_user_id,inviter_user_id,source,message,status,reject_reason,handled_by,handled_at,expires_at,version,created_at,updated_at",
From: "FROM im_group_join_requests", DefaultSort: "ORDER BY id DESC", ScopeColumn: "topic",
Filters: []internalQueryFilter{{"status", "status", "=", "string"}, {"applicant_user_id", "applicant_user_id", "=", "uid"}},
},
"invitations": {
Select: "SELECT id,invitation_id,topic,inviter_user_id,target_user_id,message,status,result,join_request_id,expires_at,handled_at,created_at,updated_at",
From: "FROM im_group_invitations", DefaultSort: "ORDER BY id DESC", ScopeColumn: "topic",
Filters: []internalQueryFilter{{"status", "status", "=", "string"}, {"target_user_id", "target_user_id", "=", "uid"}, {"inviter_user_id", "inviter_user_id", "=", "uid"}},
},
"announcements": {
Select: "SELECT id,announcement_id,topic,author_user_id,content,status,version,published_at,deleted_by,deleted_at,created_at,updated_at",
From: "FROM im_group_announcements", DefaultSort: "ORDER BY id DESC", ScopeColumn: "topic",
Filters: []internalQueryFilter{{"status", "status", "=", "string"}},
},
"mutes": {
Select: "SELECT id,topic,user_id,muted_by,muted_at,expires_at,reason,status,revoked_by,revoked_at,version,created_at,updated_at",
From: "FROM im_group_mutes", DefaultSort: "ORDER BY id DESC", ScopeColumn: "topic",
Filters: []internalQueryFilter{{"status", "status", "=", "string"}, {"user_id", "user_id", "=", "uid"}},
},
"admin-logs": {
Select: "SELECT id,log_id,topic,operator_id,target_user_id,action,object_type,object_id,before_data,after_data,request_id,idempotency_key,created_at",
From: "FROM im_group_admin_logs", DefaultSort: "ORDER BY id DESC", ScopeColumn: "topic",
Filters: []internalQueryFilter{{"action", "action", "=", "string"}, {"operator_id", "operator_id", "=", "uid"}},
},
"files": {
Select: "SELECT id,group_file_id,topic,file_id,source_topic,source_seq,uploader_id,file_name,mime_type,file_type,file_size,status,deleted_by,deleted_at,created_at,updated_at",
From: "FROM im_group_files", DefaultSort: "ORDER BY id DESC", ScopeColumn: "topic",
Filters: []internalQueryFilter{{"status", "status", "=", "string"}, {"file_type", "file_type", "=", "string"}, {"file_name", "file_name", "LIKE", "string"}, {"uploader_id", "uploader_id", "=", "uid"}},
},
}
var internalTenantResourceSpecs = map[string]*internalQuerySpec{
"audit-logs": {
Select: "SELECT id,request_id,actor_user_id,action,target_type,target_id,ip,user_agent,detail,created_at",
From: "FROM im_audit_logs", DefaultSort: "ORDER BY id DESC", DetailKey: "id",
Filters: []internalQueryFilter{{"actor_user_id", "actor_user_id", "=", "uid"}, {"action", "action", "=", "string"}, {"target_type", "target_type", "=", "string"}, {"target_id", "target_id", "=", "string"}, {"created_at_start", "created_at", ">=", "time"}, {"created_at_end", "created_at", "<=", "time"}},
},
"realname-verifications": {
Select: "SELECT id,user_id,submission_id,status,masked_name AS real_name,masked_id_number AS id_card_no_masked,review_reason AS reject_reason,verified_at AS reviewed_at,created_at,updated_at",
From: "FROM im_realname_verifications", DefaultSort: "ORDER BY id DESC", DetailKey: "id",
Filters: []internalQueryFilter{{"user_id", "user_id", "=", "uid"}, {"status", "status", "=", "string"}, {"real_name", "masked_name", "LIKE", "string"}, {"created_at_start", "created_at", ">=", "time"}, {"created_at_end", "created_at", "<=", "time"}},
},
"abuse-reports": {
Select: "SELECT id,report_id,reporter_id AS reporter_user_id,target_type,target_user_id,target_topic,target_seq,reason_code,description,evidence_file_ids,snapshot,status,created_at,updated_at",
From: "FROM im_abuse_reports", DefaultSort: "ORDER BY id DESC", DetailKey: "id",
Filters: []internalQueryFilter{{"reporter_user_id", "reporter_id", "=", "uid"}, {"target_type", "target_type", "=", "string"}, {"status", "status", "=", "string"}, {"created_at_start", "created_at", ">=", "time"}, {"created_at_end", "created_at", "<=", "time"}},
},
"feedback-submissions": {
Select: "SELECT id,feedback_id,user_id,category,content,contact,attachment_file_ids AS file_ids,client_context,include_diagnostics,status,created_at,updated_at",
From: "FROM im_feedback_submissions", DefaultSort: "ORDER BY id DESC", DetailKey: "id",
Filters: []internalQueryFilter{{"user_id", "user_id", "=", "uid"}, {"category", "category", "=", "string"}, {"status", "status", "=", "string"}, {"created_at_start", "created_at", ">=", "time"}, {"created_at_end", "created_at", "<=", "time"}},
},
"feedback-config": {
Select: "SELECT id,categories,max_length,attachment_policy,allow_contact,diagnostics_notice,created_at,updated_at",
From: "FROM im_feedback_config", DefaultSort: "ORDER BY id DESC", DetailKey: "id",
},
"mailbox-config": {
Select: "SELECT id,enabled,allow_anonymous,required_fields,max_length,attachment_policy,notice_title,notice_content,notice_updated_at,created_at,updated_at",
From: "FROM im_mailbox_config", DefaultSort: "ORDER BY id DESC", DetailKey: "id",
},
"abuse-reasons": {
Select: "SELECT id,target_type,reason_code,label,requires_description,allows_evidence,enabled,sort_order,created_at,updated_at",
From: "FROM im_abuse_reasons", DefaultSort: "ORDER BY sort_order ASC,id ASC", DetailKey: "id",
Filters: []internalQueryFilter{{"target_type", "target_type", "=", "string"}},
},
"app-releases": {
Select: "SELECT id,platform,channel,latest_version,min_supported_version,update_type,release_notes,store_url,package_url,checksum,remind_after_seconds,published_at,created_at,updated_at",
From: "FROM im_app_releases", DefaultSort: "ORDER BY id DESC", DetailKey: "id",
},
"workbench/apps": {
Select: "SELECT id,code AS app_code,name,icon_file_id,route,sort_order,unread_key,enabled,permission,created_at,updated_at",
From: "FROM im_workbench_apps", DefaultSort: "ORDER BY sort_order ASC,id ASC", DetailKey: "id",
},
}
var internalOutboxResourceSpecs = map[string]*internalQuerySpec{
"business-events": {
Select: "SELECT id,event_id,target_type,target,actor_user_id,event_type,payload,status,attempts,next_retry_at,delivered_at,created_at,updated_at",
From: "FROM im_business_events_outbox", DefaultSort: "ORDER BY id DESC", DetailKey: "id",
Filters: []internalQueryFilter{{"event_type", "event_type", "=", "string"}, {"status", "status", "=", "string"}, {"target_type", "target_type", "=", "string"}, {"target", "target", "=", "string"}, {"created_at_start", "created_at", ">=", "time"}, {"created_at_end", "created_at", "<=", "time"}},
},
"group-events": {
Select: "SELECT id,event_id,topic,actor_user_id,event_type,payload,status,attempts,next_retry_at,delivered_at,created_at,updated_at",
From: "FROM im_group_events_outbox", DefaultSort: "ORDER BY id DESC", DetailKey: "id",
Filters: []internalQueryFilter{{"event_type", "event_type", "=", "string"}, {"status", "status", "=", "string"}, {"topic", "topic", "=", "string"}, {"created_at_start", "created_at", ">=", "time"}, {"created_at_end", "created_at", "<=", "time"}},
},
}
var internalOrgResourceSpecs = map[string]*internalQuerySpec{
"departments": {
Select: "SELECT id,department_id,parent_id,name,sort_order,state,created_at,updated_at",
From: "FROM im_org_departments", DefaultSort: "ORDER BY sort_order ASC,id ASC",
Filters: []internalQueryFilter{{"parent_id", "parent_id", "=", "string"}, {"state", "state", "=", "string"}},
},
"members": {
Select: "SELECT id,department_id,user_id,display_name,title,mobile,email,employment_state,sort_order,created_at,updated_at",
From: "FROM im_org_members", DefaultSort: "ORDER BY sort_order ASC,id ASC",
Filters: []internalQueryFilter{{"department_id", "department_id", "=", "string"}, {"user_id", "user_id", "=", "uid"}, {"employment_state", "employment_state", "=", "string"}},
},
}
func serveInternalAdminResourceList(wrt http.ResponseWriter, req *http.Request, tenantID types.TenantID, resource, scope string) {
spec, ok := internalAdminQuerySpec(resource)
if !ok {
writeIMError(wrt, req, http.StatusNotFound, 30004, imReasonMalformedRequest, "resource not found", nil)
return
}
query, countQuery, args, page, perPage, err := buildInternalAdminListQuery(req, spec, scope)
if err != nil {
writeIMError(wrt, req, http.StatusBadRequest, 30000, imReasonMalformedRequest, "malformed request", nil)
return
}
result, err := store.InternalAdminQueries.Query(tenantID, query, countQuery, args, page, perPage)
if err != nil {
writeInternalQueryError(wrt, req, err)
return
}
writeIMSuccess(wrt, req, map[string]any{
"items": normalizeInternalAdminRowsForAPI(result.Items),
"total": result.Total,
"page": result.Page,
"per_page": result.PerPage,
})
}
func serveInternalAdminResourceDetail(wrt http.ResponseWriter, req *http.Request, tenantID types.TenantID, resource, id string) {
spec, ok := internalAdminQuerySpec(resource)
if !ok || spec.DetailKey == "" {
writeIMError(wrt, req, http.StatusNotFound, 30004, imReasonMalformedRequest, "resource not found", nil)
return
}
query := spec.Select + " " + spec.From + " WHERE " + spec.DetailKey + "=?"
row, err := store.InternalAdminQueries.QueryOne(tenantID, query, []any{id})
if err != nil {
writeInternalQueryError(wrt, req, err)
return
}
writeIMSuccess(wrt, req, normalizeInternalAdminRowForAPI(row))
}
func serveInternalDashboardSummary(wrt http.ResponseWriter, req *http.Request, tenantID types.TenantID) {
summary := map[string]any{}
queries := map[string]string{
"dead_kafka": "SELECT COUNT(*) AS count FROM im_kafka_dead_letters WHERE status='dead'",
"pending_business": "SELECT COUNT(*) AS count FROM im_business_events_outbox WHERE status='pending'",
"pending_group": "SELECT COUNT(*) AS count FROM im_group_events_outbox WHERE status='pending'",
"pending_realname": "SELECT COUNT(*) AS count FROM im_realname_verifications WHERE status='pending'",
"received_abuse": "SELECT COUNT(*) AS count FROM im_abuse_reports WHERE status='received'",
"received_feedback": "SELECT COUNT(*) AS count FROM im_feedback_submissions WHERE status='received'",
"active_groups": "SELECT COUNT(*) AS count FROM im_group_settings WHERE group_status='active'",
"suspended_users": "SELECT COUNT(*) AS count FROM im_users WHERE state=10",
}
for key, query := range queries {
row, err := store.InternalAdminQueries.QueryOne(tenantID, query, nil)
if err != nil {
summary[key] = nil
continue
}
summary[key] = row["count"]
}
writeIMSuccess(wrt, req, summary)
}
func serveInternalAbuseTargetQuery(wrt http.ResponseWriter, req *http.Request, tenantID types.TenantID, reportID string) {
row, err := store.InternalAdminQueries.QueryOne(tenantID,
"SELECT id,report_id,target_type,target_user_id,target_topic,target_seq,snapshot FROM im_abuse_reports WHERE report_id=?",
[]any{strings.TrimSpace(reportID)})
if err != nil {
writeInternalQueryError(wrt, req, err)
return
}
data := normalizeInternalAdminRowForAPI(row)
switch fmt.Sprint(row["target_type"]) {
case "user":
if id, ok := numericID(row["target_user_id"]); ok && id > 0 {
if user, getErr := store.InternalAdminQueries.QueryOne(tenantID, usersInternalQuerySpec().Select+" "+usersInternalQuerySpec().From+" WHERE u.id=?", []any{id}); getErr == nil {
data["target"] = normalizeInternalAdminRowForAPI(user)
}
}
case "message":
topic := fmt.Sprint(row["target_topic"])
seq, _ := numericID(row["target_seq"])
if topic != "" && seq > 0 {
if msg, getErr := store.Messages.GetOne(tenantID, topic, seq); getErr == nil {
data["target"] = map[string]any{"topic": msg.Topic, "seq_id": msg.SeqId, "from": msg.From, "head": msg.Head, "content": msg.Content, "created_at": msg.CreatedAt}
}
}
case "group":
topic := fmt.Sprint(row["target_topic"])
if topic != "" {
if group, getErr := store.InternalAdminQueries.QueryOne(tenantID, groupsInternalQuerySpec().Select+" "+groupsInternalQuerySpec().From+" WHERE t.name=?", []any{topic}); getErr == nil {
data["target"] = normalizeInternalAdminRowForAPI(group)
}
}
}
writeIMSuccess(wrt, req, data)
}
func internalAdminQuerySpec(resource string) (*internalQuerySpec, bool) {
switch resource {
case "users":
return usersInternalQuerySpec(), true
case "user_devices":
return &internalQuerySpec{Select: "SELECT id,deviceid AS device_id,platform,lastseen AS last_seen,lang", From: "FROM im_devices", DefaultSort: "ORDER BY lastseen DESC", ScopeColumn: "userid", Filters: []internalQueryFilter{{"device_id", "deviceid", "LIKE", "string"}, {"platform", "platform", "=", "string"}}}, true
case "groups":
return groupsInternalQuerySpec(), true
case "file_links":
return &internalQuerySpec{Select: "SELECT id,topic,msgid AS seq_id,userid AS user_id,createdat AS created_at", From: "FROM im_filemsglinks", DefaultSort: "ORDER BY id DESC", ScopeColumn: "fileid"}, true
case "files":
return &internalQuerySpec{Select: "SELECT id,id AS file_id,userid AS user_id,mimetype AS mime_type,size AS file_size,location,etag,status,createdat AS created_at,updatedat AS updated_at", From: "FROM im_fileuploads", DefaultSort: "ORDER BY id DESC", DetailKey: "id", Filters: []internalQueryFilter{{"file_id", "id", "=", "int"}, {"user_id", "userid", "=", "uid"}, {"status", "status", "=", "int"}, {"mime_type", "mimetype", "LIKE", "string"}, {"created_at_start", "createdat", ">=", "time"}, {"created_at_end", "createdat", "<=", "time"}}}, true
}
if strings.HasPrefix(resource, "group_") {
spec := internalGroupResourceSpecs[strings.TrimPrefix(resource, "group_")]
return spec, spec != nil
}
if strings.HasPrefix(resource, "outbox_") {
spec := internalOutboxResourceSpecs[strings.TrimPrefix(resource, "outbox_")]
return spec, spec != nil
}
if strings.HasPrefix(resource, "org_") {
spec := internalOrgResourceSpecs[strings.TrimPrefix(resource, "org_")]
return spec, spec != nil
}
spec := internalTenantResourceSpecs[resource]
return spec, spec != nil
}
func usersInternalQuerySpec() *internalQuerySpec {
return &internalQuerySpec{
Select: `SELECT u.id AS user_id,u.username,u.phone,u.state,u.stateat AS state_at,u.access,u.lastseen AS last_seen,u.useragent AS user_agent,u.public,u.trusted,
p.nickname,p.public_account,p.gender,p.region,p.avatar_file_id,p.signature,sec.auth_version,sec.password_updated_at,u.createdat AS created_at,u.updatedat AS updated_at,
(SELECT JSON_ARRAYAGG(JSON_OBJECT('method',c.method,'value',c.value,'done',c.done,'updated_at',c.updatedat)) FROM im_credentials AS c WHERE c.userid=u.id AND c.deletedat IS NULL) AS credentials`,
From: "FROM im_users AS u LEFT JOIN im_user_profiles AS p ON p.user_id=u.id LEFT JOIN im_user_security AS sec ON sec.user_id=u.id",
DefaultSort: "ORDER BY u.id DESC",
DetailKey: "u.id",
Filters: []internalQueryFilter{{"user_id", "u.id", "=", "uid"}, {"nickname", "p.nickname", "LIKE", "string"}, {"auth_name", "u.username", "LIKE", "string"}, {"state", "u.state", "=", "int"}, {"created_at_start", "u.createdat", ">=", "time"}, {"created_at_end", "u.createdat", "<=", "time"}, {"last_seen_start", "u.lastseen", ">=", "time"}, {"last_seen_end", "u.lastseen", "<=", "time"}},
}
}
func groupsInternalQuerySpec() *internalQuerySpec {
return &internalQuerySpec{
Select: `SELECT t.name AS topic,t.state,t.owner AS owner_id,t.seqid AS seq_id,t.delid AS del_id,t.subcnt AS sub_count,t.touchedat AS touched_at,t.createdat AS created_at,t.updatedat AS updated_at,t.public,t.trusted,t.access,t.tags,t.aux,
gs.name AS group_name,gs.avatar_file_id,gs.description,gs.group_status,gs.join_approval,gs.mute_all,gs.member_upload_files,gs.member_private_chat,gs.member_invite,gs.max_member_count,gs.max_admin_count,gs.member_count AS business_member_count,gs.version AS settings_version,gs.updated_at AS settings_updated_at`,
From: "FROM im_topics AS t LEFT JOIN im_group_settings AS gs ON gs.topic=t.name",
DefaultSort: "ORDER BY t.id DESC",
DetailKey: "t.name",
Filters: []internalQueryFilter{{"topic", "t.name", "LIKE", "string"}, {"owner_id", "t.owner", "=", "uid"}, {"state", "t.state", "=", "int"}, {"group_status", "gs.group_status", "=", "string"}, {"created_at_start", "t.createdat", ">=", "time"}, {"created_at_end", "t.createdat", "<=", "time"}, {"touched_at_start", "t.touchedat", ">=", "time"}, {"touched_at_end", "t.touchedat", "<=", "time"}},
}
}
func buildInternalAdminListQuery(req *http.Request, spec *internalQuerySpec, scope string) (string, string, []any, int, int, error) {
page := parseIMPositiveInt(req.URL.Query().Get("page"), 1)
perPage := parseIMLimit(firstNonEmpty(req.URL.Query().Get("per_page"), req.URL.Query().Get("limit")), 20, 200)
where := []string{"1=1"}
args := []any{}
if spec.ScopeColumn != "" && strings.TrimSpace(scope) != "" {
value, err := internalAdminFilterValue("scope", scope, scopeKindForColumn(spec.ScopeColumn))
if err != nil {
return "", "", nil, 0, 0, err
}
where = append(where, spec.ScopeColumn+"=?")
args = append(args, value)
}
for _, filter := range spec.Filters {
raw := strings.TrimSpace(req.URL.Query().Get(filter.Param))
if raw == "" {
continue
}
value, err := internalAdminFilterValue(filter.Param, raw, filter.Kind)
if err != nil {
return "", "", nil, 0, 0, err
}
op := filter.Op
if op == "" {
op = "="
}
if strings.EqualFold(op, "LIKE") {
value = "%" + fmt.Sprint(value) + "%"
}
where = append(where, filter.Column+" "+op+" ?")
args = append(args, value)
}
whereSQL := strings.Join(where, " AND ")
query := spec.Select + " " + spec.From + " WHERE " + whereSQL + " " + spec.DefaultSort
countQuery := "SELECT COUNT(*) " + spec.From + " WHERE " + whereSQL
return query, countQuery, args, page, perPage, nil
}
func internalAdminFilterValue(_ string, raw, kind string) (any, error) {
switch kind {
case "uid":
id, ok := parseInternalAdminDBID(raw)
if !ok {
return nil, types.ErrMalformed
}
return id, nil
case "int":
return strconv.ParseInt(raw, 10, 64)
default:
return raw, nil
}
}
func scopeKindForColumn(column string) string {
switch column {
case "userid", "fileid":
return "uid"
default:
return "string"
}
}
func parseIMPositiveInt(raw string, fallback int) int {
value, err := strconv.Atoi(strings.TrimSpace(raw))
if err != nil || value <= 0 {
return fallback
}
return value
}
func normalizeInternalAdminRowsForAPI(rows []map[string]any) []map[string]any {
for i := range rows {
rows[i] = normalizeInternalAdminRowForAPI(rows[i])
}
return rows
}
func normalizeInternalAdminRowForAPI(row map[string]any) map[string]any {
return row
}
func parseInternalAdminDBID(raw string) (int64, bool) {
raw = strings.TrimSpace(raw)
if raw == "" {
return 0, false
}
if id, err := strconv.ParseInt(raw, 10, 64); err == nil && id > 0 {
return id, true
}
if uid := types.ParseUserId(raw); !uid.IsZero() {
return store.DecodeUid(uid), true
}
if uid := types.ParseUid(raw); !uid.IsZero() {
return store.DecodeUid(uid), true
}
return 0, false
}
func internalAdminUIDRaw(uid types.Uid) int64 {
if uid.IsZero() {
return 0
}
return store.DecodeUid(uid)
}
func internalAdminUIDStringRaw(raw string) any {
raw = strings.TrimSpace(raw)
if raw == "" {
return nil
}
if id, ok := parseInternalAdminDBID(raw); ok {
return id
}
return raw
}
func numericID(value any) (int64, bool) {
switch v := value.(type) {
case int64:
return v, true
case int:
return int64(v), true
case int32:
return int64(v), true
case uint64:
return int64(v), true
case string:
id, err := strconv.ParseInt(v, 10, 64)
return id, err == nil
default:
return 0, false
}
}
+594
View File
@@ -0,0 +1,594 @@
package main
import (
"bytes"
"crypto/hmac"
"crypto/sha256"
"encoding/base64"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"net/url"
"strconv"
"strings"
"sync"
"time"
"github.com/tinode/chat/server/logs"
"github.com/tinode/chat/server/store"
"github.com/tinode/chat/server/store/types"
)
const (
internalCapabilityAdminQuery = "admin_query"
internalHeaderService = "X-Service-Name"
internalHeaderTimestamp = "X-Timestamp"
internalHeaderNonce = "X-Nonce"
internalHeaderSignature = "X-Signature"
)
type internalAPIConfig struct {
Enabled bool `json:"enabled"`
TimeWindowSeconds int `json:"time_window_seconds"`
Services map[string]internalAPIServiceConfig `json:"services"`
}
type internalAPIServiceConfig struct {
Secret string `json:"secret"`
Allow []string `json:"allow"`
}
type internalAPIRuntime struct {
config internalAPIConfig
nonces sync.Map
}
var internalAPI internalAPIRuntime
func initInternalAPI(raw json.RawMessage) error {
internalAPI = internalAPIRuntime{}
if len(raw) == 0 || string(raw) == "null" {
return nil
}
var cfg internalAPIConfig
if err := json.Unmarshal(raw, &cfg); err != nil {
return err
}
if cfg.TimeWindowSeconds <= 0 {
cfg.TimeWindowSeconds = 300
}
if cfg.Services == nil {
cfg.Services = map[string]internalAPIServiceConfig{}
}
internalAPI.config = cfg
if cfg.Enabled {
logs.Info.Printf("Internal API enabled for %d service(s)", len(cfg.Services))
}
return nil
}
func serveInternalAdminQuery(wrt http.ResponseWriter, req *http.Request, prefix string) {
if req.Method != http.MethodGet {
writeIMError(wrt, req, http.StatusMethodNotAllowed, 30002, imReasonMethodNotAllowed, "method not allowed", nil)
return
}
if !authorizeInternalAPI(wrt, req, internalCapabilityAdminQuery) {
return
}
tail := strings.Trim(strings.TrimPrefix(req.URL.Path, prefix), "/")
parts := splitInternalPath(tail)
if len(parts) < 3 || parts[0] != "tenants" {
writeIMError(wrt, req, http.StatusNotFound, 30004, imReasonMalformedRequest, "resource not found", nil)
return
}
tenant, ok := resolveInternalTenant(wrt, req, parts[1])
if !ok {
return
}
switch {
case len(parts) == 3 && parts[2] == "users":
serveInternalAdminResourceList(wrt, req, tenant.ID, "users", "")
case len(parts) == 4 && parts[2] == "users":
serveInternalUserQuery(wrt, req, tenant.ID, parts[3])
case len(parts) == 5 && parts[2] == "users" && parts[4] == "devices":
serveInternalAdminResourceList(wrt, req, tenant.ID, "user_devices", parts[3])
case len(parts) == 3 && parts[2] == "groups":
serveInternalAdminResourceList(wrt, req, tenant.ID, "groups", "")
case len(parts) == 4 && parts[2] == "groups":
serveInternalGroupQuery(wrt, req, tenant.ID, parts[3])
case len(parts) == 5 && parts[2] == "groups" && parts[4] == "members":
serveInternalGroupMembersQuery(wrt, req, tenant.ID, parts[3])
case len(parts) == 5 && parts[2] == "groups" && internalGroupResourceSpecs[parts[4]] != nil:
serveInternalAdminResourceList(wrt, req, tenant.ID, "group_"+parts[4], parts[3])
case len(parts) == 5 && parts[2] == "messages":
serveInternalMessageQuery(wrt, req, tenant.ID, parts[3], parts[4])
case len(parts) == 3 && parts[2] == "files":
serveInternalAdminResourceList(wrt, req, tenant.ID, "files", "")
case len(parts) == 4 && parts[2] == "files":
serveInternalFileQuery(wrt, req, tenant.ID, parts[3])
case len(parts) == 5 && parts[2] == "files" && parts[4] == "links":
serveInternalAdminResourceList(wrt, req, tenant.ID, "file_links", parts[3])
case len(parts) == 3 && internalTenantResourceSpecs[parts[2]] != nil:
serveInternalAdminResourceList(wrt, req, tenant.ID, parts[2], "")
case len(parts) == 4 && internalTenantResourceSpecs[parts[2]] != nil:
serveInternalAdminResourceDetail(wrt, req, tenant.ID, parts[2], parts[3])
case len(parts) == 4 && parts[2] == "reports" && parts[3] == "abuse":
writeIMError(wrt, req, http.StatusNotFound, 30004, imReasonMalformedRequest, "resource not found", nil)
case len(parts) == 6 && parts[2] == "reports" && parts[3] == "abuse" && parts[5] == "target":
serveInternalAbuseTargetQuery(wrt, req, tenant.ID, parts[4])
case len(parts) == 4 && parts[2] == "outbox" && internalOutboxResourceSpecs[parts[3]] != nil:
serveInternalAdminResourceList(wrt, req, tenant.ID, "outbox_"+parts[3], "")
case len(parts) == 5 && parts[2] == "outbox" && internalOutboxResourceSpecs[parts[3]] != nil:
serveInternalAdminResourceDetail(wrt, req, tenant.ID, "outbox_"+parts[3], parts[4])
case len(parts) == 4 && parts[2] == "org" && internalOrgResourceSpecs[parts[3]] != nil:
serveInternalAdminResourceList(wrt, req, tenant.ID, "org_"+parts[3], "")
case len(parts) == 4 && parts[2] == "workbench" && parts[3] == "apps":
serveInternalAdminResourceList(wrt, req, tenant.ID, "workbench/apps", "")
case len(parts) == 4 && parts[2] == "dashboard" && parts[3] == "summary":
serveInternalDashboardSummary(wrt, req, tenant.ID)
case len(parts) == 4 && parts[2] == "kafka" && parts[3] == "dead-letters":
serveInternalKafkaDeadLettersQuery(wrt, req, tenant.ID)
case len(parts) == 5 && parts[2] == "kafka" && parts[3] == "dead-letters":
serveInternalKafkaDeadLetterQuery(wrt, req, tenant.ID, parts[4])
default:
writeIMError(wrt, req, http.StatusNotFound, 30004, imReasonMalformedRequest, "resource not found", nil)
}
}
func authorizeInternalAPI(wrt http.ResponseWriter, req *http.Request, capability string) bool {
if !internalAPI.config.Enabled {
writeIMError(wrt, req, http.StatusNotFound, 30004, imReasonMalformedRequest, "resource not found", nil)
return false
}
serviceName := strings.TrimSpace(req.Header.Get(internalHeaderService))
timestampRaw := strings.TrimSpace(req.Header.Get(internalHeaderTimestamp))
nonce := strings.TrimSpace(req.Header.Get(internalHeaderNonce))
signature := strings.TrimSpace(req.Header.Get(internalHeaderSignature))
service, ok := internalAPI.config.Services[serviceName]
if !ok || service.Secret == "" || timestampRaw == "" || nonce == "" || signature == "" {
writeIMError(wrt, req, http.StatusUnauthorized, 30043, imReasonAuthRequired, "internal signature required", nil)
return false
}
if !internalServiceAllows(service, capability) {
writeIMError(wrt, req, http.StatusForbidden, 30005, imReasonPermissionDenied, "permission denied", nil)
return false
}
timestamp, err := strconv.ParseInt(timestampRaw, 10, 64)
if err != nil {
writeIMError(wrt, req, http.StatusUnauthorized, 30043, imReasonAuthRequired, "invalid internal timestamp", nil)
return false
}
window := time.Duration(internalAPI.config.TimeWindowSeconds) * time.Second
if delta := time.Since(time.Unix(timestamp, 0)); delta > window || delta < -window {
writeIMError(wrt, req, http.StatusUnauthorized, 30043, imReasonAuthRequired, "internal signature expired", nil)
return false
}
bodyHash, err := hashInternalRequestBody(req)
if err != nil {
writeIMError(wrt, req, http.StatusInternalServerError, 90000, imReasonInternal, "internal error", nil)
return false
}
signingText := strings.Join([]string{
req.Method,
req.URL.RequestURI(),
bodyHash,
timestampRaw,
nonce,
}, "\n")
if !verifyInternalSignature(service.Secret, signingText, signature) {
writeIMError(wrt, req, http.StatusUnauthorized, 30043, imReasonAuthRequired, "invalid internal signature", nil)
return false
}
if !claimInternalNonce(serviceName, nonce, time.Now().Add(window)) {
writeIMError(wrt, req, http.StatusUnauthorized, 30043, imReasonAuthRequired, "internal nonce replayed", nil)
return false
}
return true
}
func splitInternalPath(tail string) []string {
if tail == "" {
return nil
}
rawParts := strings.Split(tail, "/")
parts := make([]string, 0, len(rawParts))
for _, part := range rawParts {
if part == "" {
continue
}
decoded, err := url.PathUnescape(part)
if err != nil {
decoded = part
}
parts = append(parts, decoded)
}
return parts
}
func resolveInternalTenant(wrt http.ResponseWriter, req *http.Request, tenantCode string) (*types.Tenant, bool) {
if tenantCode == "" || store.Tenants == nil {
writeIMError(wrt, req, http.StatusBadRequest, 30000, imReasonMalformedRequest, "malformed request", map[string]any{"field": "tenant_code"})
return nil, false
}
tenant, err := store.Tenants.GetByCode(tenantCode)
if err != nil {
if errors.Is(err, types.ErrNotFound) {
writeIMError(wrt, req, http.StatusNotFound, 30010, imReasonTenantUnavailable, "tenant unavailable", nil)
return nil, false
}
writeIMError(wrt, req, http.StatusInternalServerError, 90000, imReasonInternal, "internal error", nil)
return nil, false
}
if !tenant.IsActive() {
writeIMError(wrt, req, http.StatusForbidden, 30010, imReasonTenantUnavailable, "tenant unavailable", nil)
return nil, false
}
return tenant, true
}
func serveInternalUserQuery(wrt http.ResponseWriter, req *http.Request, tenantID types.TenantID, userID string) {
userDBID, ok := parseInternalAdminDBID(userID)
if !ok {
writeIMError(wrt, req, http.StatusBadRequest, 30000, imReasonMalformedRequest, "malformed request", map[string]any{"field": "user_id"})
return
}
if spec, ok := internalAdminQuerySpec("users"); ok {
query := spec.Select + " " + spec.From + " WHERE u.id=?"
if row, err := store.InternalAdminQueries.QueryOne(tenantID, query, []any{userDBID}); err == nil {
writeIMSuccess(wrt, req, normalizeInternalAdminRowForAPI(row))
return
}
}
uid := store.EncodeUid(userDBID)
user, err := store.Users.Get(tenantID, uid)
if err != nil {
writeInternalQueryError(wrt, req, err)
return
}
writeIMSuccess(wrt, req, map[string]any{
"user_id": userDBID,
"state": user.State,
"state_at": user.StateAt,
"username": user.Username,
"phone": user.Phone,
"public": user.Public,
"trusted": user.Trusted,
"last_seen": user.LastSeen,
"user_agent": user.UserAgent,
"tags": user.Tags,
"created_at": user.CreatedAt,
"updated_at": user.UpdatedAt,
})
}
func serveInternalGroupQuery(wrt http.ResponseWriter, req *http.Request, tenantID types.TenantID, topic string) {
if topic == "" || store.GroupBusiness == nil {
writeIMError(wrt, req, http.StatusBadRequest, 30000, imReasonMalformedRequest, "malformed request", map[string]any{"field": "topic"})
return
}
if spec, ok := internalAdminQuerySpec("groups"); ok {
query := spec.Select + " " + spec.From + " WHERE t.name=?"
if row, err := store.InternalAdminQueries.QueryOne(tenantID, query, []any{topic}); err == nil {
writeIMSuccess(wrt, req, normalizeInternalAdminRowForAPI(row))
return
}
}
settings, err := store.GroupBusiness.GetSettings(tenantID, topic)
if err != nil {
writeInternalQueryError(wrt, req, err)
return
}
writeIMSuccess(wrt, req, internalGroupSettingsPayload(settings))
}
func serveInternalGroupMembersQuery(wrt http.ResponseWriter, req *http.Request, tenantID types.TenantID, topic string) {
if topic == "" || store.GroupBusiness == nil {
writeIMError(wrt, req, http.StatusBadRequest, 30000, imReasonMalformedRequest, "malformed request", map[string]any{"field": "topic"})
return
}
spec := &internalQuerySpec{
Select: `SELECT gm.id,gm.topic,gm.user_id,u.state,u.public,p.nickname,p.public_account,s.modewant AS mode_want,s.modegiven AS mode_given,s.recvseqid AS recv_seq_id,s.readseqid AS read_seq_id,s.delid AS del_id,
gm.role,gm.member_status,gm.nickname AS group_nickname,gm.pinned,gm.mute_notifications,gm.join_source,gm.joined_by,gm.joined_at,gm.left_at,gm.removed_by,gm.version,gm.created_at,gm.updated_at`,
From: "FROM im_group_members AS gm LEFT JOIN im_users AS u ON u.id=gm.user_id LEFT JOIN im_user_profiles AS p ON p.user_id=gm.user_id LEFT JOIN im_subscriptions AS s ON s.topic=gm.topic AND s.userid=gm.user_id",
DefaultSort: "ORDER BY gm.id DESC",
ScopeColumn: "gm.topic",
Filters: []internalQueryFilter{
{"user_id", "gm.user_id", "=", "uid"},
{"role", "gm.role", "=", "string"},
{"member_status", "gm.member_status", "=", "string"},
{"q", "gm.nickname", "LIKE", "string"},
{"eligible_role", "gm.role", "=", "string"},
},
}
query, countQuery, args, page, perPage, err := buildInternalAdminListQuery(req, spec, topic)
if err == nil {
if result, queryErr := store.InternalAdminQueries.Query(tenantID, query, countQuery, args, page, perPage); queryErr == nil {
writeIMSuccess(wrt, req, map[string]any{
"items": normalizeInternalAdminRowsForAPI(result.Items),
"total": result.Total,
"page": result.Page,
"per_page": result.PerPage,
"next_cursor": "",
"has_more": false,
})
return
}
}
limit := parseIMLimit(req.URL.Query().Get("limit"), 50, 200)
members, nextCursor, err := store.GroupBusiness.ListMembers(tenantID, topic, types.GroupMemberListOptions{
Role: strings.TrimSpace(req.URL.Query().Get("role")),
EligibleRole: strings.TrimSpace(req.URL.Query().Get("eligible_role")),
Query: strings.TrimSpace(req.URL.Query().Get("q")),
Cursor: strings.TrimSpace(req.URL.Query().Get("cursor")),
Limit: limit,
})
if err != nil {
writeInternalQueryError(wrt, req, err)
return
}
items := make([]any, 0, len(members))
for _, member := range members {
items = append(items, map[string]any{
"user_id": internalAdminUIDRaw(member.UserID),
"role": member.Role,
"member_status": member.MemberStatus,
"nickname": member.Nickname,
"pinned": member.Pinned,
"mute_notifications": member.MuteNotifications,
"join_source": member.JoinSource,
"joined_by": internalAdminUIDRaw(member.JoinedBy),
"joined_at": member.JoinedAt,
"left_at": member.LeftAt,
"removed_by": internalAdminUIDRaw(member.RemovedBy),
"version": member.Version,
"created_at": member.CreatedAt,
"updated_at": member.UpdatedAt,
})
}
writeIMSuccess(wrt, req, map[string]any{"items": items, "next_cursor": nextCursor})
}
func serveInternalMessageQuery(wrt http.ResponseWriter, req *http.Request, tenantID types.TenantID, topic, seqRaw string) {
seq, err := strconv.ParseInt(seqRaw, 10, 64)
if topic == "" || err != nil || seq <= 0 {
writeIMError(wrt, req, http.StatusBadRequest, 30000, imReasonMalformedRequest, "malformed request", nil)
return
}
msg, err := store.Messages.GetOne(tenantID, topic, seq)
if err != nil {
writeInternalQueryError(wrt, req, err)
return
}
writeIMSuccess(wrt, req, map[string]any{
"topic": msg.Topic,
"seq_id": msg.SeqId,
"from": internalAdminUIDStringRaw(msg.From),
"client_msg_id": msg.ClientMsgID,
"version": msg.Version,
"head": msg.Head,
"content": msg.Content,
"deleted_at": msg.DeletedAt,
"recalled_at": msg.RecalledAt,
"recalled_by": internalAdminUIDStringRaw(msg.RecalledBy),
"created_at": msg.CreatedAt,
"updated_at": msg.UpdatedAt,
})
}
func serveInternalFileQuery(wrt http.ResponseWriter, req *http.Request, tenantID types.TenantID, fileID string) {
if fileID == "" || store.Files == nil {
writeIMError(wrt, req, http.StatusBadRequest, 30000, imReasonMalformedRequest, "malformed request", map[string]any{"field": "file_id"})
return
}
file, err := store.Files.Get(tenantID, fileID)
if err != nil {
writeInternalQueryError(wrt, req, err)
return
}
writeIMSuccess(wrt, req, map[string]any{
"file_id": file.Id,
"status": file.Status,
"user_id": internalAdminUIDStringRaw(file.User),
"mime_type": file.MimeType,
"size": file.Size,
"location": file.Location,
"etag": file.ETag,
"created_at": file.CreatedAt,
"updated_at": file.UpdatedAt,
})
}
func serveInternalKafkaDeadLettersQuery(wrt http.ResponseWriter, req *http.Request, tenantID types.TenantID) {
if store.HTTPInfrastructure == nil {
writeIMError(wrt, req, http.StatusInternalServerError, 90000, imReasonInternal, "internal error", nil)
return
}
limit := parseIMLimit(req.URL.Query().Get("limit"), 50, 200)
page, err := store.HTTPInfrastructure.ListKafkaDeadLetters(tenantID, types.KafkaDeadLetterListOptions{
Status: strings.TrimSpace(req.URL.Query().Get("status")),
BizType: strings.TrimSpace(req.URL.Query().Get("biz_type")),
BizID: strings.TrimSpace(req.URL.Query().Get("biz_id")),
EventType: strings.TrimSpace(req.URL.Query().Get("event_type")),
Cursor: strings.TrimSpace(req.URL.Query().Get("cursor")),
Limit: limit,
})
if err != nil {
writeInternalQueryError(wrt, req, err)
return
}
items := make([]any, 0, len(page.Items))
for _, dead := range page.Items {
items = append(items, internalKafkaDeadLetterPayload(&dead, false))
}
writeIMSuccess(wrt, req, map[string]any{
"items": items,
"next_cursor": page.NextCursor,
"has_more": page.NextCursor != "",
})
}
func serveInternalKafkaDeadLetterQuery(wrt http.ResponseWriter, req *http.Request, tenantID types.TenantID, deadID string) {
if store.HTTPInfrastructure == nil || strings.TrimSpace(deadID) == "" {
writeIMError(wrt, req, http.StatusBadRequest, 30000, imReasonMalformedRequest, "malformed request", map[string]any{"field": "dead_id"})
return
}
dead, err := store.HTTPInfrastructure.GetKafkaDeadLetter(tenantID, deadID)
if err != nil {
writeInternalQueryError(wrt, req, err)
return
}
writeIMSuccess(wrt, req, internalKafkaDeadLetterPayload(dead, true))
}
func internalGroupSettingsPayload(settings *types.GroupSettings) map[string]any {
if settings == nil {
return nil
}
return map[string]any{
"topic": settings.Topic,
"owner_user_id": internalAdminUIDRaw(settings.OwnerUserID),
"name": settings.Name,
"avatar_file_id": settings.AvatarFileID,
"description": settings.Description,
"group_status": settings.GroupStatus,
"join_approval": settings.JoinApproval,
"mute_all": settings.MuteAll,
"member_upload_files": settings.MemberUploadFiles,
"member_private_chat": settings.MemberPrivateChat,
"member_invite": settings.MemberInvite,
"max_member_count": settings.MaxMemberCount,
"max_admin_count": settings.MaxAdminCount,
"member_count": settings.MemberCount,
"version": settings.Version,
"created_by": internalAdminUIDRaw(settings.CreatedBy),
"dissolved_by": internalAdminUIDRaw(settings.DissolvedBy),
"dissolved_at": settings.DissolvedAt,
"created_at": settings.CreatedAt,
"updated_at": settings.UpdatedAt,
}
}
func internalKafkaDeadLetterPayload(dead *types.KafkaDeadLetter, includeEnvelope bool) map[string]any {
if dead == nil {
return nil
}
payload := map[string]any{
"id": dead.ID,
"dead_id": dead.DeadID,
"event_id": dead.EventID,
"parent_event_id": dead.ParentEventID,
"source_topic": dead.SourceTopic,
"consumer_group": dead.ConsumerGroup,
"event_type": dead.EventType,
"biz_type": dead.BizType,
"biz_id": dead.BizID,
"retry_count": dead.RetryCount,
"last_reason": dead.LastReason,
"last_error": dead.LastError,
"status": dead.Status,
"replay_count": dead.ReplayCount,
"created_at": dead.CreatedAt,
"updated_at": dead.UpdatedAt,
}
if includeEnvelope {
var envelope any
if len(dead.Envelope) > 0 && json.Unmarshal(dead.Envelope, &envelope) == nil {
payload["envelope"] = envelope
} else {
payload["envelope_raw"] = string(dead.Envelope)
}
}
return payload
}
func writeInternalQueryError(wrt http.ResponseWriter, req *http.Request, err error) {
switch {
case errors.Is(err, types.ErrUserNotFound), errors.Is(err, types.ErrTopicNotFound), errors.Is(err, types.ErrNotFound):
writeIMError(wrt, req, http.StatusNotFound, 30004, imReasonMalformedRequest, "resource not found", nil)
case errors.Is(err, types.ErrMalformed):
writeIMError(wrt, req, http.StatusBadRequest, 30000, imReasonMalformedRequest, "malformed request", nil)
case errors.Is(err, types.ErrPermissionDenied):
writeIMError(wrt, req, http.StatusForbidden, 30005, imReasonPermissionDenied, "permission denied", nil)
default:
writeIMError(wrt, req, http.StatusInternalServerError, 90000, imReasonInternal, "internal error", nil)
}
}
func parseInternalUid(raw string) types.Uid {
if uid := types.ParseUserId(raw); !uid.IsZero() {
return uid
}
id, ok := parseInternalAdminDBID(raw)
if !ok {
return types.ZeroUid
}
return store.EncodeUid(id)
}
func internalServiceAllows(service internalAPIServiceConfig, capability string) bool {
for _, allowed := range service.Allow {
allowed = strings.TrimSpace(allowed)
if allowed == "*" || allowed == capability {
return true
}
}
return false
}
func hashInternalRequestBody(req *http.Request) (string, error) {
if req.Body == nil {
sum := sha256.Sum256(nil)
return hex.EncodeToString(sum[:]), nil
}
body, err := io.ReadAll(io.LimitReader(req.Body, 1<<20))
if err != nil {
return "", err
}
if err := req.Body.Close(); err != nil {
return "", err
}
req.Body = io.NopCloser(bytes.NewReader(body))
sum := sha256.Sum256(body)
return hex.EncodeToString(sum[:]), nil
}
func verifyInternalSignature(secret, signingText, signature string) bool {
signature = strings.TrimPrefix(signature, "sha256=")
mac := hmac.New(sha256.New, []byte(secret))
_, _ = mac.Write([]byte(signingText))
expected := mac.Sum(nil)
if decoded, err := hex.DecodeString(signature); err == nil {
return hmac.Equal(decoded, expected)
}
if decoded, err := base64.StdEncoding.DecodeString(signature); err == nil {
return hmac.Equal(decoded, expected)
}
return false
}
func claimInternalNonce(serviceName, nonce string, expiresAt time.Time) bool {
ttl := time.Until(expiresAt)
if ok, enabled, err := store.ClaimShortState("internal_api_nonce", fmt.Sprintf("%s:%s", serviceName, nonce), ttl); enabled {
if err != nil {
logs.Warn.Println("internal api: failed to claim nonce in redis", err)
return false
}
return ok
}
now := time.Now()
internalAPI.nonces.Range(func(key, value any) bool {
if expires, ok := value.(time.Time); ok && expires.Before(now) {
internalAPI.nonces.Delete(key)
}
return true
})
nonceKey := fmt.Sprintf("%s:%s", serviceName, nonce)
_, loaded := internalAPI.nonces.LoadOrStore(nonceKey, expiresAt)
return !loaded
}
+601
View File
@@ -0,0 +1,601 @@
package main
import (
"context"
"encoding/json"
"errors"
"strconv"
"strings"
"time"
"github.com/segmentio/kafka-go"
"github.com/tinode/chat/server/logs"
"github.com/tinode/chat/server/store"
"github.com/tinode/chat/server/store/types"
)
const (
adminCommandGroupProfileUpdate = "group.profile.update"
adminCommandGroupGovernUpdate = "group.governance.update"
adminCommandGroupAdminSet = "group.admin.set"
adminCommandGroupAdminRemove = "group.admin.remove"
adminCommandGroupMemberMute = "group.member.mute"
adminCommandGroupMemberUnmute = "group.member.unmute"
adminCommandGroupMemberRemove = "group.member.remove"
adminCommandGroupOwnerTransfer = "group.owner.transfer"
adminCommandUserBan = "user.ban"
adminCommandUserUnban = "user.unban"
adminCommandUserFreeze = "user.freeze"
adminCommandUserUnfreeze = "user.unfreeze"
adminCommandUserDeviceKick = "user.device.kick"
adminCommandUserLogoutAll = "user.logout_all"
adminCommandUserCancel = "user.account.cancel"
adminCommandUserDelete = "user.account.delete"
adminCommandKafkaDeadReplay = "kafka.dead_letter.replay"
adminCommandKafkaDeadIgnore = "kafka.dead_letter.ignore"
adminCommandKafkaDeadResolve = "kafka.dead_letter.resolve"
)
type adminCommandPayload struct {
CommandID string `json:"command_id"`
ActorUserID string `json:"actor_user_id"`
Topic string `json:"topic"`
UserID string `json:"user_id"`
UserIDs []string `json:"user_ids"`
TargetID string `json:"target_user_id"`
DeviceID string `json:"device_id"`
Version int `json:"version"`
Reason string `json:"reason"`
DeadID string `json:"dead_id"`
Note string `json:"note"`
Hard bool `json:"hard"`
ExpiresAt *time.Time `json:"expires_at"`
Profile groupProfilePatch `json:"profile"`
Governance groupGovernancePatch `json:"governance"`
}
type groupProfilePatch struct {
Name *string `json:"name"`
AvatarFileID *string `json:"avatar_file_id"`
Description *string `json:"description"`
}
type groupGovernancePatch struct {
JoinApproval *bool `json:"join_approval"`
MuteAll *bool `json:"mute_all"`
MemberUploadFiles *bool `json:"member_upload_files"`
MemberPrivateChat *bool `json:"member_private_chat"`
MemberInvite *bool `json:"member_invite"`
}
type adminCommandResultPayload struct {
CommandID string `json:"command_id"`
CommandEventID string `json:"command_event_id"`
Status string `json:"status"`
Reason string `json:"reason,omitempty"`
Message string `json:"message,omitempty"`
Result map[string]any `json:"result,omitempty"`
}
func startKafkaAdminCommandConsumer() {
startKafkaConsumer("admin_command", handleKafkaAdminCommand)
}
func handleKafkaAdminCommand(ctx context.Context, envelope kafkaEnvelope, _ kafka.Message) error {
if strings.TrimSpace(envelope.EventID) == "" {
return publishKafkaAdminCommandResult(ctx, envelope, "", "failed", "MALFORMED_EVENT", "missing event_id", nil)
}
if strings.TrimSpace(envelope.EventType) == "" {
return publishKafkaAdminCommandResult(ctx, envelope, "", "failed", "MALFORMED_EVENT", "missing event_type", nil)
}
var payload adminCommandPayload
if len(envelope.Payload) > 0 {
if err := json.Unmarshal(envelope.Payload, &payload); err != nil {
return publishKafkaAdminCommandResult(ctx, envelope, "", "failed", "MALFORMED_PAYLOAD", err.Error(), nil)
}
}
if payload.CommandID == "" {
payload.CommandID = envelope.EventID
}
tenantID, tenantErr := kafkaTenantID(envelope.TenantCode)
if tenantErr != nil {
return publishKafkaAdminCommandResult(ctx, envelope, payload.CommandID, "failed", kafkaCommandErrorReason(tenantErr), tenantErr.Error(), nil)
}
consume, claimed, beginErr := beginKafkaConsumeForTopic(tenantID, envelope, "admin_command")
if beginErr != nil {
return beginErr
}
if !claimed {
logs.Info.Printf("kafka admin command skipped: event=%s group=%s status=%s", envelope.EventID, kafkaBus.config.ConsumerGroup, consume.Status)
return nil
}
result, execErr := executeKafkaAdminCommand(tenantID, envelope, payload)
if execErr != nil {
reason := kafkaCommandErrorReason(execErr)
failedRecord, markErr := store.HTTPInfrastructure.MarkKafkaConsumeFailed(tenantID, envelope.EventID, kafkaBus.config.ConsumerGroup, reason, execErr.Error(), time.Now().UTC().Round(time.Millisecond))
if markErr != nil {
return markErr
}
if shouldDeadLetterKafkaCommand(reason, failedRecord.RetryCount) {
if deadErr := markKafkaDeadLetter(ctx, tenantID, envelope, failedRecord, reason, execErr.Error()); deadErr != nil {
return deadErr
}
if err := publishKafkaAdminCommandResult(ctx, envelope, payload.CommandID, "dead", reason, execErr.Error(), result); err != nil {
return err
}
logs.Warn.Printf("kafka admin command dead: event=%s type=%s reason=%s err=%v", envelope.EventID, envelope.EventType, reason, execErr)
return nil
}
if kafkaCommandReasonRetryable(reason) {
sleepKafkaRetryBackoff(failedRecord.RetryCount)
return execErr
}
if err := publishKafkaAdminCommandResult(ctx, envelope, payload.CommandID, "failed", reason, execErr.Error(), result); err != nil {
return err
}
logs.Warn.Printf("kafka admin command failed: event=%s type=%s reason=%s err=%v", envelope.EventID, envelope.EventType, reason, execErr)
return nil
}
if err := store.HTTPInfrastructure.MarkKafkaConsumeSucceeded(tenantID, envelope.EventID, kafkaBus.config.ConsumerGroup, time.Now().UTC().Round(time.Millisecond)); err != nil {
return err
}
if err := publishKafkaAdminCommandResult(ctx, envelope, payload.CommandID, "succeeded", "", "", result); err != nil {
return err
}
if err := enqueueKafkaAuditEvent(ctx, tenantID, "admin", parseKafkaCommandUser(payload.ActorUserID),
envelope.EventType, "admin_command", payload.CommandID, "succeeded", "", map[string]any{
"topic": payload.Topic,
"biz_id": envelope.BizID,
"event_id": envelope.EventID,
"result": result,
}); err != nil {
logs.Warn.Printf("kafka admin command audit enqueue failed: event=%s err=%v", envelope.EventID, err)
}
return nil
}
func executeKafkaAdminCommand(tenantID types.TenantID, envelope kafkaEnvelope, payload adminCommandPayload) (map[string]any, error) {
actor := parseKafkaCommandUser(payload.ActorUserID)
if actor.IsZero() {
return nil, types.ErrMalformed
}
switch strings.TrimSpace(envelope.EventType) {
case adminCommandKafkaDeadReplay:
dead, err := replayKafkaDeadLetter(context.Background(), tenantID, payload.DeadID, actor, firstNonEmpty(payload.Note, payload.Reason))
if err != nil {
return nil, err
}
return kafkaDeadLetterResult(dead), nil
case adminCommandKafkaDeadIgnore:
if err := resolveKafkaDeadLetter(tenantID, payload.DeadID, types.KafkaDeadStatusIgnored, actor, firstNonEmpty(payload.Note, payload.Reason)); err != nil {
return nil, err
}
return map[string]any{"dead_id": payload.DeadID, "status": types.KafkaDeadStatusIgnored}, nil
case adminCommandKafkaDeadResolve:
if err := resolveKafkaDeadLetter(tenantID, payload.DeadID, types.KafkaDeadStatusResolved, actor, firstNonEmpty(payload.Note, payload.Reason)); err != nil {
return nil, err
}
return map[string]any{"dead_id": payload.DeadID, "status": types.KafkaDeadStatusResolved}, nil
case adminCommandUserBan, adminCommandUserUnban, adminCommandUserFreeze, adminCommandUserUnfreeze,
adminCommandUserDeviceKick, adminCommandUserLogoutAll, adminCommandUserCancel, adminCommandUserDelete:
return executeKafkaUserGovernanceCommand(tenantID, envelope, payload, actor)
}
if store.GroupBusiness == nil {
return nil, errors.New("group business store is not initialized")
}
topic := strings.TrimSpace(payload.Topic)
if topic == "" {
return nil, types.ErrMalformed
}
now := time.Now().UTC().Round(time.Millisecond)
switch strings.TrimSpace(envelope.EventType) {
case adminCommandGroupProfileUpdate:
settings, err := store.GroupBusiness.UpdateSettings(tenantID, topic, types.GroupSettingsPatch{
Name: payload.Profile.Name,
AvatarFileID: payload.Profile.AvatarFileID,
Description: payload.Profile.Description,
Version: payload.Version,
})
if err != nil {
return nil, err
}
dispatchIMGroupEvents(tenantID)
return map[string]any{"topic": topic, "version": settings.Version, "updated_at": settings.UpdatedAt}, nil
case adminCommandGroupGovernUpdate:
settings, err := store.GroupBusiness.UpdateSettings(tenantID, topic, types.GroupSettingsPatch{
JoinApproval: payload.Governance.JoinApproval,
MuteAll: payload.Governance.MuteAll,
MemberUploadFiles: payload.Governance.MemberUploadFiles,
MemberPrivateChat: payload.Governance.MemberPrivateChat,
MemberInvite: payload.Governance.MemberInvite,
Version: payload.Version,
})
if err != nil {
return nil, err
}
dispatchIMGroupEvents(tenantID)
return map[string]any{"topic": topic, "version": settings.Version, "updated_at": settings.UpdatedAt}, nil
case adminCommandGroupAdminSet:
users := payload.UserIDs
if len(users) == 0 && payload.UserID != "" {
users = []string{payload.UserID}
}
if len(users) == 0 {
return nil, types.ErrMalformed
}
succeeded := make([]map[string]any, 0, len(users))
for _, raw := range users {
target := parseKafkaCommandUser(raw)
if target.IsZero() {
return map[string]any{"succeeded": succeeded}, types.ErrMalformed
}
if err := store.GroupBusiness.UpdateRole(tenantID, topic, actor, target,
types.GroupMemberRoleAdmin, types.ModeCAdmin|types.ModeCPublic, now); err != nil {
return map[string]any{"succeeded": succeeded, "failed_user_id": raw}, err
}
succeeded = append(succeeded, map[string]any{"user_id": internalAdminUIDRaw(target), "role": types.GroupMemberRoleAdmin})
}
dispatchIMGroupEvents(tenantID)
return map[string]any{"topic": topic, "succeeded": succeeded}, nil
case adminCommandGroupAdminRemove:
target := parseKafkaCommandUser(firstNonEmpty(payload.UserID, payload.TargetID))
if target.IsZero() {
return nil, types.ErrMalformed
}
if err := store.GroupBusiness.UpdateRole(tenantID, topic, actor, target,
types.GroupMemberRoleMember, types.ModeCPublic, now); err != nil {
return nil, err
}
dispatchIMGroupEvents(tenantID)
return map[string]any{"topic": topic, "user_id": internalAdminUIDRaw(target), "role": types.GroupMemberRoleMember}, nil
case adminCommandGroupMemberMute:
target := parseKafkaCommandUser(firstNonEmpty(payload.UserID, payload.TargetID))
if target.IsZero() {
return nil, types.ErrMalformed
}
mute, err := store.GroupBusiness.UpsertMute(tenantID, &types.GroupMute{
Topic: topic,
UserID: target,
MutedBy: actor,
MutedAt: now,
ExpiresAt: payload.ExpiresAt,
Reason: strings.TrimSpace(payload.Reason),
Status: types.GroupMuteStatusActive,
})
if err != nil {
return nil, err
}
dispatchIMGroupEvents(tenantID)
return map[string]any{"topic": topic, "user_id": internalAdminUIDRaw(target), "muted": true, "version": mute.Version}, nil
case adminCommandGroupMemberUnmute:
target := parseKafkaCommandUser(firstNonEmpty(payload.UserID, payload.TargetID))
if target.IsZero() {
return nil, types.ErrMalformed
}
if err := store.GroupBusiness.DeleteMute(tenantID, topic, target, actor, now); err != nil {
return nil, err
}
dispatchIMGroupEvents(tenantID)
return map[string]any{"topic": topic, "user_id": internalAdminUIDRaw(target), "muted": false}, nil
case adminCommandGroupMemberRemove:
target := parseKafkaCommandUser(firstNonEmpty(payload.UserID, payload.TargetID))
if target.IsZero() {
return nil, types.ErrMalformed
}
if err := store.GroupBusiness.Leave(tenantID, topic, target, now); err != nil {
return nil, err
}
dispatchIMGroupEvents(tenantID)
return map[string]any{"topic": topic, "user_id": internalAdminUIDRaw(target), "status": "left"}, nil
case adminCommandGroupOwnerTransfer:
target := parseKafkaCommandUser(firstNonEmpty(payload.TargetID, payload.UserID))
if target.IsZero() {
return nil, types.ErrMalformed
}
if err := store.GroupBusiness.TransferOwner(tenantID, topic, actor, target, payload.Version, now); err != nil {
return nil, err
}
dispatchIMGroupEvents(tenantID)
return map[string]any{"topic": topic, "owner": internalAdminUIDRaw(target), "previous_owner": internalAdminUIDRaw(actor)}, nil
}
return nil, errors.New("unsupported admin command: " + envelope.EventType)
}
func executeKafkaUserGovernanceCommand(tenantID types.TenantID, envelope kafkaEnvelope, payload adminCommandPayload, actor types.Uid) (map[string]any, error) {
target := parseKafkaCommandUser(firstNonEmpty(payload.TargetID, payload.UserID))
if target.IsZero() {
return nil, types.ErrMalformed
}
eventType := strings.TrimSpace(envelope.EventType)
now := time.Now().UTC().Round(time.Millisecond)
result := map[string]any{
"user_id": internalAdminUIDRaw(target),
"action": eventType,
}
switch eventType {
case adminCommandUserBan, adminCommandUserFreeze:
if err := store.Users.UpdateState(tenantID, target, types.StateSuspended); err != nil {
return nil, err
}
evictUserSessions(target)
notifyUserStateChanged(tenantID, target, types.StateSuspended)
result["state"] = types.StateSuspended.String()
result["evicted"] = true
return result, nil
case adminCommandUserUnban, adminCommandUserUnfreeze:
if err := store.Users.UpdateState(tenantID, target, types.StateOK); err != nil {
return nil, err
}
notifyUserStateChanged(tenantID, target, types.StateOK)
result["state"] = types.StateOK.String()
return result, nil
case adminCommandUserDeviceKick:
deviceID := strings.TrimSpace(payload.DeviceID)
if deviceID == "" {
return nil, types.ErrMalformed
}
evicted := 0
if globals.sessionStore != nil {
evicted = globals.sessionStore.EvictUserDevice(target, deviceID)
}
if store.Devices != nil {
if err := store.Devices.Delete(tenantID, target, deviceID); err != nil && !errors.Is(err, types.ErrNotFound) {
return nil, err
}
}
result["device_id"] = deviceID
result["evicted_sessions"] = evicted
return result, nil
case adminCommandUserLogoutAll:
evictUserSessions(target)
result["evicted"] = true
return result, nil
case adminCommandUserCancel, adminCommandUserDelete:
hard := payload.Hard || eventType == adminCommandUserDelete
if err := deleteUserByAdminCommand(tenantID, target, hard, now); err != nil {
return nil, err
}
result["state"] = types.StateDeleted.String()
result["hard"] = hard
return result, nil
}
return nil, errors.New("unsupported user governance command: " + eventType)
}
func evictUserSessions(uid types.Uid) {
if globals.sessionStore != nil {
globals.sessionStore.EvictUser(uid, "")
}
}
func notifyUserStateChanged(tenantID types.TenantID, uid types.Uid, state types.ObjState) {
usersRemoveUser(tenantID, uid)
if globals.hub != nil && globals.hub.userStatus != nil {
globals.hub.userStatus <- &userStatusReq{tenantID: tenantID, forUser: uid, state: state}
}
}
func deleteUserByAdminCommand(tenantID types.TenantID, uid types.Uid, hard bool, _ time.Time) error {
evictUserSessions(uid)
usersRemoveUser(tenantID, uid)
if globals.hub != nil && globals.hub.unreg != nil {
done := make(chan bool)
globals.hub.unreg <- &topicUnreg{tenantID: tenantID, forUser: uid, del: hard, done: done}
<-done
}
return store.Users.Delete(tenantID, uid, hard)
}
func publishKafkaAdminCommandResult(ctx context.Context, command kafkaEnvelope, commandID, status, reason, message string, result map[string]any) error {
if commandID == "" {
commandID = command.EventID
}
payload := adminCommandResultPayload{
CommandID: commandID,
CommandEventID: command.EventID,
Status: status,
Reason: reason,
Message: message,
Result: result,
}
encoded, err := json.Marshal(payload)
if err != nil {
return err
}
eventID := stableIMEventID("admin-command-result", command.EventID, status, reason)
return publishKafkaEnvelope(ctx, "admin_command_result", commandID, kafkaEnvelope{
EventID: eventID,
TraceID: command.TraceID,
TenantCode: command.TenantCode,
SourceService: "im-server",
TargetService: command.SourceService,
EventType: "admin.command." + status,
BizType: command.BizType,
BizID: command.BizID,
IdempotencyKey: command.IdempotencyKey,
OccurredAt: time.Now().UTC(),
ParentEventID: command.EventID,
Payload: encoded,
})
}
func kafkaTenantID(tenantCode string) (types.TenantID, error) {
tenant, err := store.Tenants.GetByCode(strings.TrimSpace(tenantCode))
if err != nil {
return types.ZeroTenantID, err
}
if tenant == nil || tenant.ID.IsZero() {
return types.ZeroTenantID, types.ErrNotFound
}
return tenant.ID, nil
}
func markKafkaDeadLetter(ctx context.Context, tenantID types.TenantID, envelope kafkaEnvelope, consume *types.KafkaConsumeRecord, reason, message string) error {
encoded, err := json.Marshal(envelope)
if err != nil {
return err
}
retryCount := 0
sourceTopic := kafkaTopic("admin_command")
if consume != nil {
retryCount = consume.RetryCount
if strings.TrimSpace(consume.SourceTopic) != "" {
sourceTopic = consume.SourceTopic
}
}
deadID := stableIMEventID("dead", envelope.EventID, kafkaBus.config.ConsumerGroup, retryCount)
dead := &types.KafkaDeadLetter{
DeadID: deadID,
EventID: envelope.EventID,
ParentEventID: envelope.ParentEventID,
SourceTopic: sourceTopic,
ConsumerGroup: kafkaBus.config.ConsumerGroup,
EventType: envelope.EventType,
BizType: envelope.BizType,
BizID: envelope.BizID,
Envelope: encoded,
RetryCount: retryCount,
LastReason: reason,
LastError: message,
Status: types.KafkaDeadStatusDead,
}
now := time.Now().UTC().Round(time.Millisecond)
if err = store.HTTPInfrastructure.MarkKafkaConsumeDead(tenantID, dead, now); err != nil {
return err
}
if auditErr := enqueueKafkaAuditEvent(ctx, tenantID, "system", types.ZeroUid, "kafka.dead_letter.created", "kafka_event", envelope.EventID, "dead", reason, map[string]any{
"dead_id": deadID,
"event_type": envelope.EventType,
"biz_type": envelope.BizType,
"biz_id": envelope.BizID,
"consumer_group": kafkaBus.config.ConsumerGroup,
}); auditErr != nil {
logs.Warn.Printf("kafka dead letter audit enqueue failed: event=%s dead=%s err=%v", envelope.EventID, deadID, auditErr)
}
deadPayload, err := json.Marshal(dead)
if err != nil {
return err
}
return publishKafkaEnvelope(ctx, "dead_letter", envelope.EventID, kafkaEnvelope{
EventID: deadID,
TraceID: envelope.TraceID,
TenantCode: envelope.TenantCode,
SourceService: "im-server",
TargetService: "hotgo-admin",
EventType: "kafka.dead_letter.created",
BizType: envelope.BizType,
BizID: envelope.BizID,
OccurredAt: now,
ParentEventID: envelope.EventID,
Payload: deadPayload,
})
}
func shouldDeadLetterKafkaCommand(reason string, retryCount int) bool {
if !kafkaCommandReasonRetryable(reason) {
return true
}
if kafkaBus == nil {
return true
}
return retryCount >= kafkaBus.config.MaxRetryCount
}
func kafkaCommandReasonRetryable(reason string) bool {
switch reason {
case "COMMAND_FAILED":
return true
default:
return false
}
}
func sleepKafkaRetryBackoff(retryCount int) {
if kafkaBus == nil || retryCount <= 0 {
return
}
backoffs := kafkaBus.config.RetryBackoffMS
index := retryCount - 1
if index >= len(backoffs) {
index = len(backoffs) - 1
}
delay := time.Duration(backoffs[index]) * time.Millisecond
if delay > 0 {
time.Sleep(delay)
}
}
func parseKafkaCommandUser(raw string) types.Uid {
raw = strings.TrimSpace(raw)
if raw == "" {
return types.ZeroUid
}
if id, err := strconv.ParseInt(raw, 10, 64); err == nil && id > 0 {
return store.EncodeUid(id)
}
uid := types.ParseUserId(raw)
if !uid.IsZero() {
return uid
}
return types.ParseUid(raw)
}
func kafkaDeadLetterResult(dead *types.KafkaDeadLetter) map[string]any {
if dead == nil {
return nil
}
return map[string]any{
"dead_id": dead.DeadID,
"event_id": dead.EventID,
"source_topic": dead.SourceTopic,
"event_type": dead.EventType,
"biz_type": dead.BizType,
"biz_id": dead.BizID,
"status": dead.Status,
"replay_count": dead.ReplayCount,
"updated_at": dead.UpdatedAt,
}
}
func kafkaCommandErrorReason(err error) string {
if err == nil {
return ""
}
switch {
case errors.Is(err, types.ErrMalformed):
return "MALFORMED_COMMAND"
case errors.Is(err, types.ErrNotFound):
return "RESOURCE_NOT_FOUND"
case errors.Is(err, types.ErrPermissionDenied):
return "FORBIDDEN"
case errors.Is(err, types.ErrDuplicate):
return "COMMAND_DUPLICATED"
case errors.Is(err, types.ErrInvalidResponse):
return "GROUP_VERSION_CONFLICT"
default:
if strings.Contains(err.Error(), "unsupported admin command") {
return "UNSUPPORTED_COMMAND"
}
return "COMMAND_FAILED"
}
}
func firstNonEmpty(values ...string) string {
for _, value := range values {
if strings.TrimSpace(value) != "" {
return strings.TrimSpace(value)
}
}
return ""
}
+130
View File
@@ -0,0 +1,130 @@
package main
import (
"context"
"encoding/json"
"strings"
"time"
"github.com/segmentio/kafka-go"
"github.com/tinode/chat/server/logs"
"github.com/tinode/chat/server/store"
"github.com/tinode/chat/server/store/types"
)
const (
cacheInvalidateEventAll = "cache.invalidate"
cacheInvalidateEventTenant = "cache.tenant.invalidate"
cacheInvalidateEventRegistration = "cache.registration.invalidate"
cacheInvalidateEventThirdParty = "cache.third_party.invalidate"
cacheInvalidateEventDictionary = "cache.dictionary.invalidate"
cacheInvalidateEventRedisObject = "cache.redis_object.invalidate"
cacheInvalidateEventGroupHotMessages = "cache.group_hot_messages.invalidate"
cacheInvalidateNamespaceTenant = "tenant"
cacheInvalidateNamespaceRegistration = "registration"
cacheInvalidateNamespaceThirdParty = "third_party"
cacheInvalidateNamespaceDictionary = "dictionary"
cacheInvalidateNamespaceGroupHotMessage = "group_hot_messages"
)
type cacheInvalidatePayload struct {
TenantCode string `json:"tenant_code"`
TenantID types.TenantID `json:"tenant_id"`
Namespace string `json:"namespace"`
Group string `json:"group"`
Key string `json:"key"`
Keys []string `json:"keys"`
Topic string `json:"topic"`
Ranges []types.Range `json:"ranges"`
}
func startKafkaCacheInvalidateConsumer() {
startKafkaConsumer("cache_invalidate", handleKafkaCacheInvalidateEvent)
}
func handleKafkaCacheInvalidateEvent(ctx context.Context, envelope kafkaEnvelope, _ kafka.Message) error {
if strings.TrimSpace(envelope.EventID) == "" {
return nil
}
tenantID, _ := kafkaTenantIDFromEnvelopeOrPayload(envelope)
if tenantID.IsZero() {
tenantID = types.TenantID(1)
}
consume, claimed, err := beginKafkaConsumeForTopic(tenantID, envelope, "cache_invalidate")
if err != nil {
return err
}
if !claimed {
logs.Info.Printf("kafka cache invalidate skipped: event=%s group=%s status=%s", envelope.EventID, kafkaBus.config.ConsumerGroup, consume.Status)
return nil
}
if err = executeKafkaCacheInvalidateEvent(envelope); err != nil {
reason := kafkaCommandErrorReason(err)
failed, markErr := store.HTTPInfrastructure.MarkKafkaConsumeFailed(tenantID, envelope.EventID, kafkaBus.config.ConsumerGroup, reason, err.Error(), time.Now().UTC().Round(time.Millisecond))
if markErr != nil {
return markErr
}
if failed.RetryCount >= kafkaBus.config.MaxRetryCount {
return markKafkaDeadLetter(ctx, tenantID, envelope, failed, reason, err.Error())
}
sleepKafkaRetryBackoff(failed.RetryCount)
return err
}
return store.HTTPInfrastructure.MarkKafkaConsumeSucceeded(tenantID, envelope.EventID, kafkaBus.config.ConsumerGroup, time.Now().UTC().Round(time.Millisecond))
}
func executeKafkaCacheInvalidateEvent(envelope kafkaEnvelope) error {
var payload cacheInvalidatePayload
if len(envelope.Payload) > 0 {
if err := json.Unmarshal(envelope.Payload, &payload); err != nil {
return err
}
}
tenantCode := firstNonEmpty(payload.TenantCode, envelope.TenantCode)
tenantID := payload.TenantID
if tenantID.IsZero() && tenantCode != "" {
if tenant, err := store.Tenants.GetByCode(tenantCode); err == nil && tenant != nil {
tenantID = tenant.ID
}
}
eventType := strings.TrimSpace(envelope.EventType)
namespace := strings.TrimSpace(payload.Namespace)
switch {
case eventType == cacheInvalidateEventTenant || namespace == cacheInvalidateNamespaceTenant:
store.InvalidateTenantConfigCache(tenantCode, tenantID)
case eventType == cacheInvalidateEventRegistration || namespace == cacheInvalidateNamespaceRegistration:
store.InvalidateRegistrationConfigCache()
case eventType == cacheInvalidateEventThirdParty || eventType == cacheInvalidateEventDictionary ||
namespace == cacheInvalidateNamespaceThirdParty || namespace == cacheInvalidateNamespaceDictionary:
store.InvalidateThirdPartyConfigCache(payload.Group)
case eventType == cacheInvalidateEventGroupHotMessages || namespace == cacheInvalidateNamespaceGroupHotMessage:
if tenantID.IsValid() && strings.TrimSpace(payload.Topic) != "" {
if err := store.InvalidateHotMessages(tenantID, strings.TrimSpace(payload.Topic), payload.Ranges); err != nil {
return err
}
}
case eventType == cacheInvalidateEventRedisObject || namespace != "":
if tenantID.IsValid() && namespace != "" {
keys := append([]string{}, payload.Keys...)
if strings.TrimSpace(payload.Key) != "" {
keys = append(keys, strings.TrimSpace(payload.Key))
}
for _, key := range keys {
key = strings.TrimSpace(key)
if key == "" {
continue
}
if err := store.DeleteCachedObject(tenantID, namespace, key); err != nil {
return err
}
}
}
default:
store.InvalidateTenantConfigCache(tenantCode, tenantID)
store.InvalidateThirdPartyConfigCache(payload.Group)
store.InvalidateRegistrationConfigCache()
}
logs.Info.Printf("kafka cache invalidate applied: event=%s type=%s tenant=%s namespace=%s group=%s", envelope.EventID, envelope.EventType, tenantCode, namespace, payload.Group)
return nil
}
+122
View File
@@ -0,0 +1,122 @@
package main
import (
"context"
"encoding/json"
"strings"
"time"
"github.com/segmentio/kafka-go"
"github.com/tinode/chat/server/logs"
"github.com/tinode/chat/server/store"
"github.com/tinode/chat/server/store/types"
)
const (
configEventTenantUpdated = "tenant.updated"
configEventTenantStatusChanged = "tenant.status.changed"
configEventTenantLimitsUpdated = "tenant.limits.updated"
configEventTenantFeaturesUpdated = "tenant.features.updated"
configEventDictionaryUpdated = "dictionary.updated"
configEventThirdPartyUpdated = "third_party.updated"
configEventRegistrationUpdated = "registration.config.updated"
)
type configEventPayload struct {
TenantCode string `json:"tenant_code"`
TenantID types.TenantID `json:"tenant_id"`
Group string `json:"group"`
Keys []string `json:"keys"`
}
func startKafkaConfigEventConsumer() {
startKafkaConsumer("config_event", handleKafkaConfigEvent)
}
func handleKafkaConfigEvent(ctx context.Context, envelope kafkaEnvelope, _ kafka.Message) error {
if strings.TrimSpace(envelope.EventID) == "" {
return nil
}
tenantID, _ := kafkaTenantIDFromEnvelopeOrPayload(envelope)
if tenantID.IsZero() {
tenantID = types.TenantID(1)
}
consume, claimed, err := beginKafkaConsumeForTopic(tenantID, envelope, "config_event")
if err != nil {
return err
}
if !claimed {
logs.Info.Printf("kafka config event skipped: event=%s group=%s status=%s", envelope.EventID, kafkaBus.config.ConsumerGroup, consume.Status)
return nil
}
if err = executeKafkaConfigEvent(envelope); err != nil {
reason := kafkaCommandErrorReason(err)
failed, markErr := store.HTTPInfrastructure.MarkKafkaConsumeFailed(tenantID, envelope.EventID, kafkaBus.config.ConsumerGroup, reason, err.Error(), time.Now().UTC().Round(time.Millisecond))
if markErr != nil {
return markErr
}
if failed.RetryCount >= kafkaBus.config.MaxRetryCount {
return markKafkaDeadLetter(ctx, tenantID, envelope, failed, reason, err.Error())
}
sleepKafkaRetryBackoff(failed.RetryCount)
return err
}
return store.HTTPInfrastructure.MarkKafkaConsumeSucceeded(tenantID, envelope.EventID, kafkaBus.config.ConsumerGroup, time.Now().UTC().Round(time.Millisecond))
}
func executeKafkaConfigEvent(envelope kafkaEnvelope) error {
var payload configEventPayload
if len(envelope.Payload) > 0 {
if err := json.Unmarshal(envelope.Payload, &payload); err != nil {
return err
}
}
tenantCode := firstNonEmpty(payload.TenantCode, envelope.TenantCode)
tenantID := payload.TenantID
if tenantID.IsZero() && tenantCode != "" {
if tenant, err := store.Tenants.GetByCode(tenantCode); err == nil && tenant != nil {
tenantID = tenant.ID
}
}
switch strings.TrimSpace(envelope.EventType) {
case configEventTenantUpdated, configEventTenantStatusChanged, configEventTenantLimitsUpdated, configEventTenantFeaturesUpdated:
store.InvalidateTenantConfigCache(tenantCode, tenantID)
case configEventDictionaryUpdated, configEventThirdPartyUpdated:
store.InvalidateThirdPartyConfigCache(payload.Group)
case configEventRegistrationUpdated:
store.InvalidateRegistrationConfigCache()
default:
store.InvalidateTenantConfigCache(tenantCode, tenantID)
store.InvalidateThirdPartyConfigCache(payload.Group)
store.InvalidateRegistrationConfigCache()
}
logs.Info.Printf("kafka config event applied: event=%s type=%s tenant=%s group=%s", envelope.EventID, envelope.EventType, tenantCode, payload.Group)
return nil
}
func kafkaTenantIDFromEnvelopeOrPayload(envelope kafkaEnvelope) (types.TenantID, error) {
var payload configEventPayload
if len(envelope.Payload) > 0 {
_ = json.Unmarshal(envelope.Payload, &payload)
}
if payload.TenantID.IsValid() {
return payload.TenantID, nil
}
return kafkaTenantID(firstNonEmpty(envelope.TenantCode, payload.TenantCode))
}
func beginKafkaConsumeForTopic(tenantID types.TenantID, envelope kafkaEnvelope, topicName string) (*types.KafkaConsumeRecord, bool, error) {
if kafkaBus == nil {
return nil, false, types.ErrUnsupported
}
deadline := time.Now().UTC().Add(5 * time.Minute).Round(time.Millisecond)
return store.HTTPInfrastructure.BeginKafkaConsume(tenantID, &types.KafkaConsumeRecord{
EventID: envelope.EventID,
ConsumerGroup: kafkaBus.config.ConsumerGroup,
SourceTopic: kafkaTopic(topicName),
EventType: envelope.EventType,
BizType: envelope.BizType,
BizID: envelope.BizID,
}, deadline)
}
+41
View File
@@ -0,0 +1,41 @@
package main
import (
"context"
"encoding/json"
"strings"
"time"
"github.com/tinode/chat/server/store"
"github.com/tinode/chat/server/store/types"
)
func replayKafkaDeadLetter(ctx context.Context, tenantID types.TenantID, deadID string, actor types.Uid, note string) (*types.KafkaDeadLetter, error) {
now := time.Now().UTC().Round(time.Millisecond)
dead, err := store.HTTPInfrastructure.MarkKafkaDeadLetterReplay(tenantID, deadID, actor.UserId(), note, now)
if err != nil {
return nil, err
}
var envelope kafkaEnvelope
if err = json.Unmarshal(dead.Envelope, &envelope); err != nil {
return nil, err
}
if envelope.EventID == "" {
envelope.EventID = dead.EventID
}
if envelope.ParentEventID == "" {
envelope.ParentEventID = dead.ParentEventID
}
if err = publishKafkaEnvelope(ctx, strings.TrimSpace(dead.SourceTopic), envelope.EventID, envelope); err != nil {
return nil, err
}
if err = store.HTTPInfrastructure.MarkKafkaDeadLetter(tenantID, deadID, types.KafkaDeadStatusReplayed, actor.UserId(), note, now); err != nil {
return nil, err
}
return store.HTTPInfrastructure.GetKafkaDeadLetter(tenantID, deadID)
}
func resolveKafkaDeadLetter(tenantID types.TenantID, deadID, status string, actor types.Uid, note string) error {
now := time.Now().UTC().Round(time.Millisecond)
return store.HTTPInfrastructure.MarkKafkaDeadLetter(tenantID, deadID, status, actor.UserId(), note, now)
}
+265
View File
@@ -0,0 +1,265 @@
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"strings"
"time"
"github.com/segmentio/kafka-go"
"github.com/tinode/chat/server/logs"
"github.com/tinode/chat/server/store"
"github.com/tinode/chat/server/store/types"
)
const (
kafkaFileLifecycleUploadCompleted = "file.upload.completed"
kafkaFileLifecycleAttachmentsBind = "file.attachments.bind"
kafkaFileLifecycleGroupFileCreate = "group.file.created"
kafkaFileLifecycleGroupFileDelete = "group.file.deleted"
)
type kafkaFileLifecyclePayload struct {
Action string `json:"action"`
FileID string `json:"file_id,omitempty"`
FileIDs []string `json:"file_ids,omitempty"`
AttachmentURLs []string `json:"attachment_urls,omitempty"`
Topic string `json:"topic,omitempty"`
MessageID string `json:"message_id,omitempty"`
SeqID int64 `json:"seq_id,omitempty"`
UserID string `json:"user_id,omitempty"`
GroupFileID string `json:"group_file_id,omitempty"`
FileType string `json:"file_type,omitempty"`
FileName string `json:"file_name,omitempty"`
MIMEType string `json:"mime_type,omitempty"`
FileSize int64 `json:"file_size,omitempty"`
Storage map[string]any `json:"storage,omitempty"`
OccurredAt time.Time `json:"occurred_at"`
}
func startKafkaFileLifecycleConsumer() {
startKafkaConsumer("file_lifecycle", handleKafkaFileLifecycle)
}
func enqueueKafkaFileUploaded(ctx context.Context, tenantID types.TenantID, fd *types.FileDef) error {
if !kafkaEnabled() || fd == nil || fd.TenantID != tenantID || fd.Status != types.UploadCompleted {
return nil
}
payload := kafkaFileLifecyclePayload{
Action: kafkaFileLifecycleUploadCompleted,
FileID: fd.Uid().String(),
UserID: types.ParseUid(fd.User).UserId(),
MIMEType: fd.MimeType,
FileSize: fd.Size,
OccurredAt: fd.UpdatedAt,
Storage: map[string]any{
"location": fd.Location,
"etag": fd.ETag,
},
}
return enqueueKafkaFileLifecycle(ctx, tenantID, kafkaFileLifecycleUploadCompleted, fd.Uid().String(), payload)
}
func enqueueKafkaFileAttachmentsBind(ctx context.Context, tenantID types.TenantID, msg *types.Message, attachmentURLs []string) error {
if !kafkaEnabled() || msg == nil || msg.TenantID != tenantID || len(attachmentURLs) == 0 {
return nil
}
payload := kafkaFileLifecyclePayload{
Action: kafkaFileLifecycleAttachmentsBind,
AttachmentURLs: append([]string(nil), attachmentURLs...),
Topic: msg.Topic,
MessageID: msg.Uid().String(),
SeqID: msg.SeqId,
UserID: types.ParseUid(msg.From).UserId(),
FileType: lifecycleFileTypeFromMessageHead(msg.Head),
OccurredAt: msg.CreatedAt,
}
return enqueueKafkaFileLifecycle(ctx, tenantID, kafkaFileLifecycleAttachmentsBind, fmt.Sprintf("%s:%d", msg.Topic, msg.SeqId), payload)
}
func enqueueKafkaGroupFileLifecycle(ctx context.Context, tenantID types.TenantID, eventType string, file *types.GroupFile, actor types.Uid) error {
if !kafkaEnabled() || file == nil || file.Topic == "" || file.FileID == "" {
return nil
}
occurredAt := file.UpdatedAt
if occurredAt.IsZero() {
occurredAt = time.Now().UTC().Round(time.Millisecond)
}
payload := kafkaFileLifecyclePayload{
Action: eventType,
FileID: file.FileID,
FileIDs: []string{file.FileID},
Topic: file.Topic,
GroupFileID: file.GroupFileID,
SeqID: file.SourceSeq,
UserID: actor.UserId(),
FileType: file.FileType,
FileName: file.FileName,
MIMEType: file.MIMEType,
FileSize: file.FileSize,
OccurredAt: occurredAt,
}
return enqueueKafkaFileLifecycle(ctx, tenantID, eventType, firstNonEmpty(file.GroupFileID, file.FileID), payload)
}
func enqueueKafkaFileLifecycle(_ context.Context, tenantID types.TenantID, eventType, bizID string, payload kafkaFileLifecyclePayload) error {
tenantCode := ""
if store.Tenants != nil {
if tenant, err := store.Tenants.GetByID(tenantID); err == nil && tenant != nil {
tenantCode = tenant.Code
} else {
logs.Warn.Printf("kafka file lifecycle: failed to resolve tenant code for tenant %d: %v", tenantID, err)
}
}
if payload.OccurredAt.IsZero() {
payload.OccurredAt = time.Now().UTC().Round(time.Millisecond)
}
encoded, err := json.Marshal(payload)
if err != nil {
return err
}
eventID := stableIMEventID("file-lifecycle", tenantID, eventType, bizID, payload.SeqID, strings.Join(payload.AttachmentURLs, ","))
err = enqueueKafkaOutbox(tenantID, "file_lifecycle", bizID, kafkaEnvelope{
EventID: eventID,
TenantCode: tenantCode,
SourceService: "im-server",
TargetService: "im-server",
EventType: eventType,
BizType: "file",
BizID: bizID,
OccurredAt: payload.OccurredAt,
Payload: encoded,
})
if err != nil {
return err
}
actor := types.ParseUserId(payload.UserID)
if auditErr := enqueueKafkaAuditEvent(context.Background(), tenantID, "user", actor, eventType, "file", bizID, "succeeded", "", map[string]any{
"file_id": payload.FileID,
"file_ids": payload.FileIDs,
"topic": payload.Topic,
"group_file_id": payload.GroupFileID,
"seq_id": payload.SeqID,
}); auditErr != nil {
logs.Warn.Printf("kafka file lifecycle: failed to enqueue audit event type=%s biz=%s: %v", eventType, bizID, auditErr)
}
return nil
}
func handleKafkaFileLifecycle(ctx context.Context, envelope kafkaEnvelope, _ kafka.Message) error {
tenantID, err := kafkaTenantID(envelope.TenantCode)
if err != nil {
return err
}
consume, claimed, err := beginKafkaConsumeForTopic(tenantID, envelope, "file_lifecycle")
if err != nil || !claimed {
return err
}
var payload kafkaFileLifecyclePayload
if err = json.Unmarshal(envelope.Payload, &payload); err != nil {
return markKafkaDeadLetter(ctx, tenantID, envelope, consume, "MALFORMED_PAYLOAD", err.Error())
}
if err = executeKafkaFileLifecycle(tenantID, payload); err != nil {
failed, markErr := store.HTTPInfrastructure.MarkKafkaConsumeFailed(tenantID, envelope.EventID, kafkaBus.config.ConsumerGroup,
kafkaCommandErrorReason(err), err.Error(), time.Now().UTC().Round(time.Millisecond))
if markErr != nil {
return markErr
}
if failed != nil && failed.RetryCount >= kafkaBus.config.MaxRetryCount {
return markKafkaDeadLetter(ctx, tenantID, envelope, failed, kafkaCommandErrorReason(err), err.Error())
}
sleepKafkaRetryBackoff(failed.RetryCount)
return err
}
return store.HTTPInfrastructure.MarkKafkaConsumeSucceeded(tenantID, envelope.EventID, kafkaBus.config.ConsumerGroup, time.Now().UTC().Round(time.Millisecond))
}
func executeKafkaFileLifecycle(tenantID types.TenantID, payload kafkaFileLifecyclePayload) error {
switch payload.Action {
case kafkaFileLifecycleAttachmentsBind:
return executeKafkaFileAttachmentsBind(tenantID, payload)
case kafkaFileLifecycleUploadCompleted, kafkaFileLifecycleGroupFileCreate, kafkaFileLifecycleGroupFileDelete:
return nil
default:
return types.ErrMalformed
}
}
func executeKafkaFileAttachmentsBind(tenantID types.TenantID, payload kafkaFileLifecyclePayload) error {
if payload.Topic == "" || payload.MessageID == "" || len(payload.AttachmentURLs) == 0 {
return types.ErrMalformed
}
msgID := types.ParseUid(payload.MessageID)
if msgID.IsZero() {
return types.ErrMalformed
}
if types.GetTopicCat(payload.Topic) != types.TopicCatGrp {
return store.Files.LinkAttachments(tenantID, "", msgID, payload.AttachmentURLs)
}
uploader := types.ParseUserId(payload.UserID)
if uploader.IsZero() {
return types.ErrMalformed
}
for _, url := range payload.AttachmentURLs {
fid := store.Store.GetMediaHandler().GetIdFromUrl(url)
if fid.IsZero() {
continue
}
_, err := store.GroupBusiness.CreateFile(tenantID, &types.GroupFile{
Topic: payload.Topic,
FileID: fid.String(),
SourceTopic: payload.Topic,
SourceSeq: payload.SeqID,
UploaderID: uploader,
FileType: lifecycleFileType(payload.FileType),
IdempotencyKey: fmt.Sprintf("%s:%d:%s", payload.Topic, payload.SeqID, fid.String()),
})
if err != nil && !errors.Is(err, types.ErrDuplicate) {
return err
}
}
return nil
}
func lifecycleFileTypeFromMessageHead(head types.KVMap) string {
msgType := lifecycleHeaderString(head, "x-im-type")
if msgType == "" {
msgType = lifecycleHeaderString(head, "type")
}
if msgType == "" {
return "file"
}
return lifecycleFileType(msgType)
}
func lifecycleFileType(raw string) string {
switch lifecycleCanonicalMessageType(raw) {
case "image", "audio", "voice", "video":
return lifecycleCanonicalMessageType(raw)
default:
return "file"
}
}
func lifecycleHeaderString(head types.KVMap, key string) string {
if head == nil {
return ""
}
value, ok := head[key]
if !ok {
return ""
}
str, _ := value.(string)
return strings.TrimSpace(str)
}
func lifecycleCanonicalMessageType(raw string) string {
switch strings.ToLower(strings.TrimSpace(raw)) {
case "audio":
return "voice"
default:
return strings.ToLower(strings.TrimSpace(raw))
}
}
+153
View File
@@ -0,0 +1,153 @@
package main
import (
"context"
"encoding/json"
"errors"
"fmt"
"time"
"github.com/segmentio/kafka-go"
"github.com/tinode/chat/server/logs"
"github.com/tinode/chat/server/store"
"github.com/tinode/chat/server/store/types"
)
type kafkaMessagePersistPayload struct {
Message types.Message `json:"message"`
AttachmentURLs []string `json:"attachment_urls,omitempty"`
ReadBySender bool `json:"read_by_sender,omitempty"`
MessageKind string `json:"message_kind"`
}
func startKafkaMessagePersistConsumer() {
startKafkaConsumer("message_persist", handleKafkaMessagePersist)
}
func saveMessageMaybeKafka(ctx context.Context, tenantID types.TenantID, msg *types.Message, attachmentURLs []string, readBySender bool) (*types.MessageSaveResult, error) {
if !kafkaEnabled() {
return store.Messages.Save(tenantID, msg, attachmentURLs, readBySender)
}
saveResult, err := store.PrepareMessagePersist(tenantID, msg, attachmentURLs, readBySender)
if err != nil || saveResult == nil || saveResult.Duplicate {
return saveResult, err
}
if readBySender && !types.ParseUid(msg.From).IsZero() {
saveResult.MarkedReadBySender = true
}
if err = enqueueKafkaMessagePersist(ctx, tenantID, msg, attachmentURLs, readBySender); err != nil {
return nil, err
}
return saveResult, nil
}
func enqueueKafkaMessagePersist(_ context.Context, tenantID types.TenantID, msg *types.Message, attachmentURLs []string, readBySender bool) error {
tenantCode := ""
if store.Tenants != nil {
if tenant, err := store.Tenants.GetByID(tenantID); err == nil && tenant != nil {
tenantCode = tenant.Code
} else {
logs.Warn.Printf("kafka message persist: failed to resolve tenant code for tenant %d: %v", tenantID, err)
}
}
kind := "p2p"
if types.GetTopicCat(msg.Topic) == types.TopicCatGrp {
kind = "group"
}
payload, err := json.Marshal(kafkaMessagePersistPayload{
Message: *msg,
AttachmentURLs: attachmentURLs,
ReadBySender: readBySender,
MessageKind: kind,
})
if err != nil {
return err
}
eventID := fmt.Sprintf("msg.persist.%d.%s.%d", tenantID, msg.Topic, msg.SeqId)
envelope := kafkaEnvelope{
EventID: eventID,
TenantCode: tenantCode,
SourceService: "im-server",
TargetService: "im-server",
EventType: "message.persist." + kind,
BizType: "message",
BizID: fmt.Sprintf("%s:%d", msg.Topic, msg.SeqId),
IdempotencyKey: msg.ClientMsgID,
Version: msg.SeqId,
OccurredAt: msg.CreatedAt,
Actor: &kafkaActor{Type: "user", ID: types.ParseUid(msg.From).UserId()},
Payload: payload,
}
return enqueueKafkaOutbox(tenantID, "message_persist", msg.Topic, envelope)
}
func handleKafkaMessagePersist(ctx context.Context, envelope kafkaEnvelope, _ kafka.Message) error {
tenantID, err := kafkaTenantID(envelope.TenantCode)
if err != nil {
return err
}
consume, claimed, err := beginKafkaConsumeForTopic(tenantID, envelope, "message_persist")
if err != nil || !claimed {
return err
}
var payload kafkaMessagePersistPayload
if err = json.Unmarshal(envelope.Payload, &payload); err != nil {
return markKafkaDeadLetter(ctx, tenantID, envelope, consume, "MALFORMED_PAYLOAD", err.Error())
}
if payload.Message.TenantID == types.ZeroTenantID {
payload.Message.TenantID = tenantID
}
if payload.Message.TenantID != tenantID {
return markKafkaDeadLetter(ctx, tenantID, envelope, consume, "TENANT_MISMATCH", "message tenant does not match envelope tenant")
}
if err = validateKafkaMessageKind(payload); err != nil {
return markKafkaDeadLetter(ctx, tenantID, envelope, consume, "MALFORMED_PAYLOAD", err.Error())
}
if _, err = store.PersistPreparedMessage(tenantID, &payload.Message, nil, payload.ReadBySender); err != nil && !errors.Is(err, types.ErrDuplicate) {
failed, markErr := store.HTTPInfrastructure.MarkKafkaConsumeFailed(tenantID, envelope.EventID, kafkaBus.config.ConsumerGroup,
kafkaCommandErrorReason(err), err.Error(), time.Now().UTC().Round(time.Millisecond))
if markErr != nil {
return markErr
}
if failed != nil && failed.RetryCount >= kafkaBus.config.MaxRetryCount {
return markKafkaDeadLetter(ctx, tenantID, envelope, failed, kafkaCommandErrorReason(err), err.Error())
}
sleepKafkaRetryBackoff(failed.RetryCount)
return err
}
if len(payload.AttachmentURLs) > 0 {
if err = enqueueKafkaFileAttachmentsBind(ctx, tenantID, &payload.Message, payload.AttachmentURLs); err != nil {
failed, markErr := store.HTTPInfrastructure.MarkKafkaConsumeFailed(tenantID, envelope.EventID, kafkaBus.config.ConsumerGroup,
"KAFKA_FILE_LIFECYCLE_FAILED", err.Error(), time.Now().UTC().Round(time.Millisecond))
if markErr != nil {
return markErr
}
if failed != nil && failed.RetryCount >= kafkaBus.config.MaxRetryCount {
return markKafkaDeadLetter(ctx, tenantID, envelope, failed, "KAFKA_FILE_LIFECYCLE_FAILED", err.Error())
}
sleepKafkaRetryBackoff(failed.RetryCount)
return err
}
}
return store.HTTPInfrastructure.MarkKafkaConsumeSucceeded(tenantID, envelope.EventID, kafkaBus.config.ConsumerGroup, time.Now().UTC().Round(time.Millisecond))
}
func validateKafkaMessageKind(payload kafkaMessagePersistPayload) error {
if payload.Message.Topic == "" || payload.Message.SeqId <= 0 {
return types.ErrMalformed
}
isGroup := types.GetTopicCat(payload.Message.Topic) == types.TopicCatGrp
switch payload.MessageKind {
case "group":
if !isGroup {
return types.ErrMalformed
}
case "p2p", "":
if isGroup {
return types.ErrMalformed
}
default:
return types.ErrMalformed
}
return nil
}
+126
View File
@@ -0,0 +1,126 @@
package main
import (
"context"
"encoding/json"
"time"
"github.com/segmentio/kafka-go"
"github.com/tinode/chat/server/logs"
"github.com/tinode/chat/server/store"
"github.com/tinode/chat/server/store/types"
)
const (
kafkaOutboxPollInterval = 2 * time.Second
kafkaOutboxTenantLimit = 1000
kafkaOutboxBatchLimit = 100
)
func startKafkaOutboxPoller() {
if !kafkaEnabled() {
return
}
kafkaBus.wg.Add(1)
go func() {
defer kafkaBus.wg.Done()
ticker := time.NewTicker(kafkaOutboxPollInterval)
defer ticker.Stop()
for {
select {
case <-kafkaBus.ctx.Done():
return
default:
drainKafkaOutbox(kafkaBus.ctx)
}
select {
case <-kafkaBus.ctx.Done():
return
case <-ticker.C:
}
}
}()
}
func enqueueKafkaOutbox(tenantID types.TenantID, topicName, key string, envelope kafkaEnvelope) error {
if envelope.OccurredAt.IsZero() {
envelope.OccurredAt = time.Now().UTC()
}
raw, err := json.Marshal(envelope)
if err != nil {
return err
}
return store.HTTPInfrastructure.CreateKafkaOutbox(tenantID, &types.KafkaOutboxRecord{
EventID: envelope.EventID,
Topic: topicName,
EventType: envelope.EventType,
BizType: envelope.BizType,
BizID: envelope.BizID,
Key: key,
Envelope: raw,
Status: types.KafkaOutboxPending,
NextRetryAt: nil,
})
}
func drainKafkaOutbox(ctx context.Context) {
if store.Tenants == nil || store.HTTPInfrastructure == nil {
return
}
tenants, err := store.Tenants.ListActive(kafkaOutboxTenantLimit)
if err != nil {
logs.Warn.Printf("kafka outbox: list active tenants: %v", err)
return
}
now := time.Now().UTC().Round(time.Millisecond)
for _, tenant := range tenants {
if tenant.ID.IsZero() {
continue
}
records, err := store.HTTPInfrastructure.ListPendingKafkaOutbox(tenant.ID, kafkaOutboxBatchLimit, now)
if err != nil {
logs.Warn.Printf("kafka outbox: list tenant=%d: %v", tenant.ID, err)
continue
}
for _, record := range records {
if err = publishKafkaOutboxRecord(ctx, record); err != nil {
nextRetryAt := now.Add(kafkaOutboxRetryDelay(record.Attempts + 1))
if markErr := store.HTTPInfrastructure.MarkKafkaOutboxFailed(tenant.ID, record.EventID,
"KAFKA_PUBLISH_FAILED", err.Error(), nextRetryAt, now); markErr != nil {
logs.Warn.Printf("kafka outbox: mark failed event=%s tenant=%d: %v", record.EventID, tenant.ID, markErr)
}
continue
}
if err = store.HTTPInfrastructure.MarkKafkaOutboxSent(tenant.ID, record.EventID, time.Now().UTC().Round(time.Millisecond)); err != nil {
logs.Warn.Printf("kafka outbox: mark sent event=%s tenant=%d: %v", record.EventID, tenant.ID, err)
}
}
}
}
func publishKafkaOutboxRecord(ctx context.Context, record types.KafkaOutboxRecord) error {
topic := kafkaTopic(record.Topic)
if topic == "" {
topic = record.Topic
}
return kafkaBus.writer.WriteMessages(ctx, kafka.Message{
Topic: topic,
Key: []byte(record.Key),
Value: record.Envelope,
Time: time.Now().UTC(),
})
}
func kafkaOutboxRetryDelay(attempt int) time.Duration {
if attempt <= 0 {
attempt = 1
}
if kafkaBus == nil || len(kafkaBus.config.RetryBackoffMS) == 0 {
return time.Duration(attempt) * 10 * time.Second
}
index := attempt - 1
if index >= len(kafkaBus.config.RetryBackoffMS) {
index = len(kafkaBus.config.RetryBackoffMS) - 1
}
return time.Duration(kafkaBus.config.RetryBackoffMS[index]) * time.Millisecond
}
+308
View File
@@ -0,0 +1,308 @@
package main
import (
"context"
"crypto/tls"
"encoding/json"
"errors"
"strings"
"sync"
"time"
"github.com/segmentio/kafka-go"
"github.com/tinode/chat/server/logs"
)
const (
kafkaTopicAdminCommand = "im.admin.command"
kafkaTopicAdminCommandResult = "im.admin.command.result"
kafkaTopicConfigEvent = "im.config.event"
kafkaTopicBusinessEvent = "im.business.event"
kafkaTopicMessagePersist = "im.message.persist"
kafkaTopicMessagePush = "im.message.push"
kafkaTopicFileSecurity = "im.file.security"
kafkaTopicFileLifecycle = "im.file.lifecycle"
kafkaTopicWorkbenchEvent = "im.workbench.event"
kafkaTopicCacheInvalidate = "im.cache.invalidate"
kafkaTopicAuditEvent = "im.audit.event"
kafkaTopicDeadLetter = "im.dead.letter"
)
type kafkaConfig struct {
Enabled bool `json:"enabled"`
Brokers []string `json:"brokers"`
ClientID string `json:"client_id"`
ConsumerGroup string `json:"consumer_group"`
Topics map[string]string `json:"topics"`
DialTimeoutMS int `json:"dial_timeout_ms"`
WriteTimeoutMS int `json:"write_timeout_ms"`
ReadTimeoutMS int `json:"read_timeout_ms"`
RequiredAcks int `json:"required_acks"`
AllowAutoTopic bool `json:"allow_auto_topic_creation"`
TLSEnabled bool `json:"tls_enabled"`
MaxRetryCount int `json:"max_retry_count"`
RetryBackoffMS []int `json:"retry_backoff_ms"`
DeadRetentionSec int `json:"dead_letter_retention_seconds"`
}
type kafkaEnvelope struct {
EventID string `json:"event_id"`
TraceID string `json:"trace_id,omitempty"`
TenantCode string `json:"tenant_code,omitempty"`
SourceService string `json:"source_service,omitempty"`
TargetService string `json:"target_service,omitempty"`
EventType string `json:"event_type"`
BizType string `json:"biz_type,omitempty"`
BizID string `json:"biz_id,omitempty"`
IdempotencyKey string `json:"idempotency_key,omitempty"`
Version int64 `json:"version,omitempty"`
OccurredAt time.Time `json:"occurred_at"`
Actor *kafkaActor `json:"actor,omitempty"`
Payload json.RawMessage `json:"payload,omitempty"`
ParentEventID string `json:"parent_event_id,omitempty"`
}
type kafkaActor struct {
Type string `json:"type,omitempty"`
ID string `json:"id,omitempty"`
DisplayName string `json:"display_name,omitempty"`
}
type kafkaRuntime struct {
config kafkaConfig
topics map[string]string
writer *kafka.Writer
ctx context.Context
cancel context.CancelFunc
wg sync.WaitGroup
}
var kafkaBus *kafkaRuntime
func initKafkaRuntime(raw json.RawMessage) error {
if len(raw) == 0 {
kafkaBus = nil
logs.Info.Println("Kafka disabled: no config")
return nil
}
var cfg kafkaConfig
if err := json.Unmarshal(raw, &cfg); err != nil {
return err
}
if !cfg.Enabled {
kafkaBus = nil
logs.Info.Println("Kafka disabled")
return nil
}
normalizeKafkaConfig(&cfg)
if len(cfg.Brokers) == 0 {
return errors.New("kafka: enabled but brokers are not configured")
}
ctx, cancel := context.WithCancel(context.Background())
rt := &kafkaRuntime{
config: cfg,
topics: kafkaTopics(cfg.Topics),
ctx: ctx,
cancel: cancel,
}
transport := &kafka.Transport{
ClientID: cfg.ClientID,
}
if cfg.TLSEnabled {
transport.TLS = &tls.Config{MinVersion: tls.VersionTLS12}
}
rt.writer = &kafka.Writer{
Addr: kafka.TCP(cfg.Brokers...),
Balancer: &kafka.Hash{},
Transport: transport,
RequiredAcks: kafka.RequiredAcks(cfg.RequiredAcks),
AllowAutoTopicCreation: cfg.AllowAutoTopic,
WriteTimeout: time.Duration(cfg.WriteTimeoutMS) * time.Millisecond,
ReadTimeout: time.Duration(cfg.ReadTimeoutMS) * time.Millisecond,
}
if err := rt.ping(context.Background()); err != nil {
cancel()
_ = rt.writer.Close()
return err
}
kafkaBus = rt
logs.Info.Printf("Kafka enabled: brokers=%s client_id=%s group=%s", strings.Join(cfg.Brokers, ","), cfg.ClientID, cfg.ConsumerGroup)
return nil
}
func normalizeKafkaConfig(cfg *kafkaConfig) {
if cfg.ClientID == "" {
cfg.ClientID = "im-server"
}
if cfg.ConsumerGroup == "" {
cfg.ConsumerGroup = "im-server"
}
if cfg.DialTimeoutMS <= 0 {
cfg.DialTimeoutMS = 3000
}
if cfg.WriteTimeoutMS <= 0 {
cfg.WriteTimeoutMS = 3000
}
if cfg.ReadTimeoutMS <= 0 {
cfg.ReadTimeoutMS = 3000
}
if cfg.RequiredAcks == 0 {
cfg.RequiredAcks = int(kafka.RequireOne)
}
if cfg.MaxRetryCount <= 0 {
cfg.MaxRetryCount = 5
}
if len(cfg.RetryBackoffMS) == 0 {
cfg.RetryBackoffMS = []int{10000, 30000, 60000, 300000, 900000}
}
if cfg.DeadRetentionSec <= 0 {
cfg.DeadRetentionSec = 90 * 24 * 3600
}
}
func kafkaTopics(overrides map[string]string) map[string]string {
topics := map[string]string{
"admin_command": kafkaTopicAdminCommand,
"admin_command_result": kafkaTopicAdminCommandResult,
"config_event": kafkaTopicConfigEvent,
"business_event": kafkaTopicBusinessEvent,
"message_persist": kafkaTopicMessagePersist,
"message_push": kafkaTopicMessagePush,
"file_security": kafkaTopicFileSecurity,
"file_lifecycle": kafkaTopicFileLifecycle,
"workbench_event": kafkaTopicWorkbenchEvent,
"cache_invalidate": kafkaTopicCacheInvalidate,
"audit_event": kafkaTopicAuditEvent,
"dead_letter": kafkaTopicDeadLetter,
}
for key, value := range overrides {
if value != "" {
topics[key] = value
}
}
return topics
}
func (rt *kafkaRuntime) ping(ctx context.Context) error {
timeout := time.Duration(rt.config.DialTimeoutMS) * time.Millisecond
dialer := rt.dialer()
pingCtx, cancel := context.WithTimeout(ctx, timeout)
defer cancel()
conn, err := dialer.DialContext(pingCtx, "tcp", rt.config.Brokers[0])
if err != nil {
return err
}
return conn.Close()
}
func kafkaEnabled() bool {
return kafkaBus != nil
}
func kafkaTopic(name string) string {
if kafkaBus == nil {
return ""
}
return kafkaBus.topics[name]
}
func publishKafkaEnvelope(ctx context.Context, topicName string, key string, envelope kafkaEnvelope) error {
if kafkaBus == nil {
return nil
}
if envelope.OccurredAt.IsZero() {
envelope.OccurredAt = time.Now().UTC()
}
topic := kafkaBus.topics[topicName]
if topic == "" {
topic = topicName
}
value, err := json.Marshal(envelope)
if err != nil {
return err
}
return kafkaBus.writer.WriteMessages(ctx, kafka.Message{
Topic: topic,
Key: []byte(key),
Value: value,
Time: envelope.OccurredAt,
})
}
type kafkaMessageHandler func(context.Context, kafkaEnvelope, kafka.Message) error
func startKafkaConsumer(topicName string, handler kafkaMessageHandler) {
if kafkaBus == nil || handler == nil {
return
}
topic := kafkaBus.topics[topicName]
if topic == "" {
topic = topicName
}
reader := kafka.NewReader(kafka.ReaderConfig{
Brokers: kafkaBus.config.Brokers,
GroupID: kafkaBus.config.ConsumerGroup,
Topic: topic,
StartOffset: kafka.FirstOffset,
MinBytes: 1,
MaxBytes: 10e6,
MaxWait: time.Second,
Dialer: kafkaBus.dialer(),
})
kafkaBus.wg.Add(1)
go func() {
defer kafkaBus.wg.Done()
defer reader.Close()
for {
msg, err := reader.FetchMessage(kafkaBus.ctx)
if err != nil {
if errors.Is(err, context.Canceled) || errors.Is(kafkaBus.ctx.Err(), context.Canceled) {
return
}
logs.Err.Println("Kafka fetch failed:", err)
continue
}
var envelope kafkaEnvelope
if err = json.Unmarshal(msg.Value, &envelope); err != nil {
logs.Err.Println("Kafka envelope decode failed:", err)
_ = reader.CommitMessages(context.Background(), msg)
continue
}
if err = handler(kafkaBus.ctx, envelope, msg); err != nil {
logs.Err.Println("Kafka handler failed:", envelope.EventID, envelope.EventType, err)
continue
}
if err = reader.CommitMessages(context.Background(), msg); err != nil {
logs.Err.Println("Kafka commit failed:", err)
}
}
}()
logs.Info.Printf("Kafka consumer started: topic=%s group=%s", topic, kafkaBus.config.ConsumerGroup)
}
func (rt *kafkaRuntime) dialer() *kafka.Dialer {
dialer := &kafka.Dialer{
ClientID: rt.config.ClientID,
Timeout: time.Duration(rt.config.DialTimeoutMS) * time.Millisecond,
}
if rt.config.TLSEnabled {
dialer.TLS = &tls.Config{MinVersion: tls.VersionTLS12}
}
return dialer
}
func stopKafkaRuntime() {
if kafkaBus == nil {
return
}
kafkaBus.cancel()
kafkaBus.wg.Wait()
if kafkaBus.writer != nil {
_ = kafkaBus.writer.Close()
}
logs.Info.Println("Stopped Kafka runtime")
kafkaBus = nil
}
+171
View File
@@ -0,0 +1,171 @@
package main
import (
"context"
"encoding/json"
"time"
"github.com/segmentio/kafka-go"
"github.com/tinode/chat/server/logs"
"github.com/tinode/chat/server/store"
"github.com/tinode/chat/server/store/types"
)
type kafkaWorkbenchPayload struct {
Module string `json:"module"`
Action string `json:"action"`
ResourceID string `json:"resource_id"`
ActorUserID string `json:"actor_user_id"`
TargetUserIDs []string `json:"target_user_ids,omitempty"`
Title string `json:"title,omitempty"`
Summary string `json:"summary,omitempty"`
Data map[string]any `json:"data,omitempty"`
OccurredAt time.Time `json:"occurred_at"`
}
type kafkaAuditPayload struct {
ActorType string `json:"actor_type"`
ActorID string `json:"actor_id"`
Action string `json:"action"`
Resource string `json:"resource"`
ResourceID string `json:"resource_id"`
Result string `json:"result"`
Reason string `json:"reason,omitempty"`
Data map[string]any `json:"data,omitempty"`
OccurredAt time.Time `json:"occurred_at"`
}
func startKafkaWorkbenchEventConsumer() {
startKafkaConsumer("workbench_event", handleKafkaWorkbenchEvent)
}
func enqueueKafkaWorkbenchEvent(ctx context.Context, tenantID types.TenantID, actor types.Uid, module, action, resourceID string,
targets []types.Uid, title, summary string, data map[string]any) error {
if !kafkaEnabled() {
return nil
}
targetUserIDs := make([]string, 0, len(targets))
for _, target := range targets {
if !target.IsZero() {
targetUserIDs = append(targetUserIDs, target.UserId())
}
}
payload := kafkaWorkbenchPayload{
Module: module,
Action: action,
ResourceID: resourceID,
ActorUserID: actor.UserId(),
TargetUserIDs: targetUserIDs,
Title: title,
Summary: summary,
Data: data,
OccurredAt: time.Now().UTC().Round(time.Millisecond),
}
return enqueueKafkaTypedEvent(ctx, tenantID, "workbench_event", "workbench."+action, "workbench", resourceID, actor, payload)
}
func enqueueKafkaAuditEvent(ctx context.Context, tenantID types.TenantID, actorType string, actor types.Uid,
action, resource, resourceID, result, reason string, data map[string]any) error {
if !kafkaEnabled() {
return nil
}
actorID := ""
if !actor.IsZero() {
actorID = actor.UserId()
}
payload := kafkaAuditPayload{
ActorType: actorType,
ActorID: actorID,
Action: action,
Resource: resource,
ResourceID: resourceID,
Result: result,
Reason: reason,
Data: data,
OccurredAt: time.Now().UTC().Round(time.Millisecond),
}
return enqueueKafkaTypedEvent(ctx, tenantID, "audit_event", "audit."+action, "audit", resource+":"+resourceID, actor, payload)
}
func enqueueKafkaTypedEvent(_ context.Context, tenantID types.TenantID, topicName, eventType, bizType, bizID string, actor types.Uid, payload any) error {
tenantCode := ""
if store.Tenants != nil {
if tenant, err := store.Tenants.GetByID(tenantID); err == nil && tenant != nil {
tenantCode = tenant.Code
} else {
logs.Warn.Printf("kafka event: failed to resolve tenant code for tenant %d: %v", tenantID, err)
}
}
encoded, err := json.Marshal(payload)
if err != nil {
return err
}
eventID := stableIMEventID(topicName, tenantID, eventType, bizID, encoded)
envelope := kafkaEnvelope{
EventID: eventID,
TenantCode: tenantCode,
SourceService: "im-server",
TargetService: "im-server",
EventType: eventType,
BizType: bizType,
BizID: bizID,
OccurredAt: time.Now().UTC(),
Payload: encoded,
}
if !actor.IsZero() {
envelope.Actor = &kafkaActor{Type: "user", ID: actor.UserId()}
}
return enqueueKafkaOutbox(tenantID, topicName, bizID, envelope)
}
func handleKafkaWorkbenchEvent(ctx context.Context, envelope kafkaEnvelope, _ kafka.Message) error {
tenantID, err := kafkaTenantID(envelope.TenantCode)
if err != nil {
return err
}
consume, claimed, err := beginKafkaConsumeForTopic(tenantID, envelope, "workbench_event")
if err != nil || !claimed {
return err
}
var payload kafkaWorkbenchPayload
if err = json.Unmarshal(envelope.Payload, &payload); err != nil {
return markKafkaDeadLetter(ctx, tenantID, envelope, consume, "MALFORMED_PAYLOAD", err.Error())
}
if err = executeKafkaWorkbenchEvent(tenantID, envelope, payload); err != nil {
failed, markErr := store.HTTPInfrastructure.MarkKafkaConsumeFailed(tenantID, envelope.EventID, kafkaBus.config.ConsumerGroup,
kafkaCommandErrorReason(err), err.Error(), time.Now().UTC().Round(time.Millisecond))
if markErr != nil {
return markErr
}
if failed != nil && failed.RetryCount >= kafkaBus.config.MaxRetryCount {
return markKafkaDeadLetter(ctx, tenantID, envelope, failed, kafkaCommandErrorReason(err), err.Error())
}
sleepKafkaRetryBackoff(failed.RetryCount)
return err
}
return store.HTTPInfrastructure.MarkKafkaConsumeSucceeded(tenantID, envelope.EventID, kafkaBus.config.ConsumerGroup, time.Now().UTC().Round(time.Millisecond))
}
func executeKafkaWorkbenchEvent(tenantID types.TenantID, envelope kafkaEnvelope, payload kafkaWorkbenchPayload) error {
actor := types.ParseUserId(payload.ActorUserID)
if actor.IsZero() {
actor = types.ParseUid(payload.ActorUserID)
}
for _, raw := range payload.TargetUserIDs {
target := types.ParseUserId(raw)
if target.IsZero() {
continue
}
eventID := stableIMEventID("workbench", envelope.EventID, target.UserId())
if target != actor && payload.Title != "" {
if err := createIMSystemNotification(tenantID, target, stableIMEventID(eventID, "notification"), payload.Action,
payload.Title, payload.Summary, types.KVMap{"module": payload.Module, "resource_id": payload.ResourceID}); err != nil {
return err
}
}
if err := enqueueIMBusinessEvent(tenantID, eventID, "user", target.UserId(), actor, payload.Action, payload.Data); err != nil {
return err
}
}
return nil
}
+26
View File
@@ -324,6 +324,9 @@ type configType struct {
Cluster json.RawMessage `json:"cluster_config"`
Plugin json.RawMessage `json:"plugins"`
Store json.RawMessage `json:"store_config"`
Kafka json.RawMessage `json:"kafka"`
Internal json.RawMessage `json:"internal_api"`
AdminAPI json.RawMessage `json:"admin_internal_api"`
Push json.RawMessage `json:"push"`
TLS json.RawMessage `json:"tls"`
Auth map[string]json.RawMessage `json:"auth_config"`
@@ -450,6 +453,17 @@ func main() {
}()
statsRegisterDbStats()
if err = initKafkaRuntime(config.Kafka); err != nil {
logs.Err.Fatal("Failed to initialize Kafka:", err)
}
defer stopKafkaRuntime()
if err = initInternalAPI(config.Internal); err != nil {
logs.Err.Fatal("Failed to initialize internal API:", err)
}
if err = initAdminInternalAPI(config.AdminAPI); err != nil {
logs.Err.Fatal("Failed to initialize Admin internal API client:", err)
}
// API key signing secret
globals.apiKeySalt = config.APIKeySalt
err = store.InitAuthLogicalNames(config.Auth["logical_names"])
@@ -678,6 +692,13 @@ func main() {
// The hub (the main message router)
globals.hub = newHub()
startRedisGroupFanout(context.Background())
startKafkaAdminCommandConsumer()
startKafkaConfigEventConsumer()
startKafkaCacheInvalidateConsumer()
startKafkaMessagePersistConsumer()
startKafkaFileLifecycleConsumer()
startKafkaWorkbenchEventConsumer()
startKafkaOutboxPoller()
stopExternalBusinessEvents := startIMExternalBusinessEventPoller()
defer stopExternalBusinessEvents()
@@ -832,6 +853,7 @@ func main() {
mux.HandleFunc(config.ApiPath+"v1/im/search/conversations", serveIMConversationSearch)
mux.HandleFunc(config.ApiPath+"v1/im/conversations/p2p", serveIMP2PConversation)
mux.HandleFunc(config.ApiPath+"v1/im/forward-targets", serveIMForwardTargets)
mux.HandleFunc(config.ApiPath+"v1/im/messages/forward", serveIMMessageForward)
mux.HandleFunc(config.ApiPath+"v1/im/system-notifications", serveIMSystemNotifications)
mux.HandleFunc(config.ApiPath+"v1/im/system-notifications/", serveIMSystemNotificationRead)
mux.HandleFunc(config.ApiPath+"v1/im/favorites", serveIMFavorites)
@@ -852,6 +874,10 @@ func main() {
mux.HandleFunc(config.ApiPath+"v1/im/blocks/", serveIMBlockUser)
mux.HandleFunc(config.ApiPath+"v1/im/friends", serveIMFriends)
mux.HandleFunc(config.ApiPath+"v1/im/friends/", serveIMFriendResource)
adminQueryPrefix := config.ApiPath + "internal/admin-query/"
mux.HandleFunc(adminQueryPrefix, func(wrt http.ResponseWriter, req *http.Request) {
serveInternalAdminQuery(wrt, req, adminQueryPrefix)
})
// Handle long polling clients. Enable compression.
mux.Handle(config.ApiPath+"v0/channels/lp", gh.CompressHandler(http.HandlerFunc(serveLongPoll)))
if config.Media != nil {
+7
View File
@@ -27,6 +27,13 @@ func (s testTenantStore) GetByID(types.TenantID) (*types.Tenant, error) {
return s.tenant, s.err
}
func (s testTenantStore) ListActive(int) ([]types.Tenant, error) {
if s.err != nil || s.tenant == nil {
return nil, s.err
}
return []types.Tenant{*s.tenant}, nil
}
func test_makeSession(uid types.Uid) *Session {
return &Session{
send: make(chan any, 10),
+27
View File
@@ -241,6 +241,33 @@ func (ss *SessionStore) EvictUser(uid types.Uid, skipSid string) {
statsSet("LiveSessions", int64(len(ss.sessCache)))
}
// EvictUserDevice terminates all sessions of a given user connected with a device.
func (ss *SessionStore) EvictUserDevice(uid types.Uid, deviceID string) int {
ss.lock.Lock()
defer ss.lock.Unlock()
if uid.IsZero() || deviceID == "" {
return 0
}
evicted := NoErrEvicted("", "", types.TimeNow())
evicted.AsUser = uid.UserId()
count := 0
for _, s := range ss.sessCache {
if s.uid == uid && s.deviceID == deviceID && !s.isMultiplex() {
_, data := s.serialize(evicted)
s.stopSession(data)
delete(ss.sessCache, s.sid)
if s.proto == LPOLL {
ss.lru.Remove(s.lpTracker)
}
count++
}
}
statsSet("LiveSessions", int64(len(ss.sessCache)))
return count
}
// NodeRestarted removes stale sessions from a restarted cluster node.
// - nodeName is the name of affected node
// - fingerprint is the new fingerprint of the node.
+53
View File
@@ -2,6 +2,7 @@ package store
import (
"sort"
"strconv"
"strings"
"sync"
"time"
@@ -42,6 +43,18 @@ func (c *configCache[T]) set(key string, value T) {
c.mu.Unlock()
}
func (c *configCache[T]) delete(key string) {
c.mu.Lock()
delete(c.items, key)
c.mu.Unlock()
}
func (c *configCache[T]) clear() {
c.mu.Lock()
c.items = make(map[string]configCacheEntry[T])
c.mu.Unlock()
}
var (
tenantByCodeCache = newConfigCache[*types.Tenant]()
tenantByIDCache = newConfigCache[*types.Tenant]()
@@ -83,3 +96,43 @@ func thirdPartyCacheKey(group string, keys ...string) string {
sort.Strings(normalized)
return group + ":" + strings.Join(normalized, ",")
}
// InvalidateTenantConfigCache clears cached tenant registry/config records. An
// empty selector clears all tenant records.
func InvalidateTenantConfigCache(code string, id types.TenantID) {
if strings.TrimSpace(code) != "" {
tenantByCodeCache.delete(strings.TrimSpace(code))
} else {
tenantByCodeCache.clear()
}
if id.IsValid() {
tenantByIDCache.delete(formatTenantIDForCache(id))
} else if strings.TrimSpace(code) == "" {
tenantByIDCache.clear()
}
}
// InvalidateRegistrationConfigCache clears cached global registration config.
func InvalidateRegistrationConfigCache() {
registrationSMSCache.clear()
}
// InvalidateThirdPartyConfigCache clears cached third-party config.
func InvalidateThirdPartyConfigCache(group string) {
if strings.TrimSpace(group) == "" {
thirdPartyConfigCache.clear()
return
}
prefix := strings.TrimSpace(group) + ":"
thirdPartyConfigCache.mu.Lock()
for key := range thirdPartyConfigCache.items {
if strings.HasPrefix(key, prefix) {
delete(thirdPartyConfigCache.items, key)
}
}
thirdPartyConfigCache.mu.Unlock()
}
func formatTenantIDForCache(id types.TenantID) string {
return strconv.FormatInt(int64(id), 10)
}
+27
View File
@@ -69,6 +69,7 @@ type groupMessageRuntime interface {
RemoveCachedSetMember(tenantID types.TenantID, namespace, id, member string) error
CachedSetContains(tenantID types.TenantID, namespace, id, member string) (bool, error)
AllowRate(tenantID types.TenantID, scope string, limit int, window time.Duration) (bool, int, time.Duration, error)
ClaimShortState(namespace, id string, ttl time.Duration) (bool, error)
}
var groupMsgRuntime groupMessageRuntime
@@ -88,6 +89,14 @@ func MarkUserOnline(tenantID types.TenantID, uid types.Uid, sid, nodeID, device,
return groupMsgRuntime.MarkUserOnline(tenantID, uid, sid, nodeID, device, userAgent)
}
func ClaimShortState(namespace, id string, ttl time.Duration) (bool, bool, error) {
if groupMsgRuntime == nil || namespace == "" || id == "" {
return false, false, nil
}
ok, err := groupMsgRuntime.ClaimShortState(namespace, id, ttl)
return ok, true, err
}
func RefreshUserPresence(tenantID types.TenantID, uid types.Uid, sid, nodeID string) error {
if groupMsgRuntime == nil || !tenantID.IsValid() || uid.IsZero() || sid == "" {
return nil
@@ -169,6 +178,13 @@ func DeleteCachedObject(tenantID types.TenantID, namespace, id string) error {
return groupMsgRuntime.DeleteCachedObject(tenantID, namespace, id)
}
func InvalidateHotMessages(tenantID types.TenantID, topic string, ranges []types.Range) error {
if groupMsgRuntime == nil || !tenantID.IsValid() || topic == "" {
return nil
}
return groupMsgRuntime.InvalidateHotMessages(tenantID, topic, ranges)
}
func AddCachedSetMember(tenantID types.TenantID, namespace, id, member string) error {
if groupMsgRuntime == nil || !tenantID.IsValid() || namespace == "" || id == "" || member == "" {
return nil
@@ -730,6 +746,13 @@ func (r *redisGroupMessageRuntime) AllowRate(tenantID types.TenantID, scope stri
return count <= int64(limit), int(count), ttl, nil
}
func (r *redisGroupMessageRuntime) ClaimShortState(namespace, id string, ttl time.Duration) (bool, error) {
if ttl <= 0 {
ttl = time.Duration(r.cfg.ShortTTL) * time.Second
}
return r.client.SetNX(context.Background(), r.shortStateKey(namespace, id), "1", ttl).Result()
}
func (r *redisGroupMessageRuntime) hotSeqIDs(tenantID types.TenantID, topic string, opts *types.QueryOpt, limit int) ([]int64, bool, error) {
if opts != nil && len(opts.IdRanges) > 0 {
seqs := expandRanges(opts.IdRanges, limit)
@@ -851,6 +874,10 @@ func (r *redisGroupMessageRuntime) rateKey(tenantID types.TenantID, scope string
return r.key(tenantID, "rate", scope)
}
func (r *redisGroupMessageRuntime) shortStateKey(namespace, id string) string {
return r.key(types.ZeroTenantID, "short", namespace, id)
}
func (r *redisGroupMessageRuntime) cacheTTL(namespace string) time.Duration {
ttl := r.cfg.CacheTTL
if namespace == "verification" || namespace == "action_token" {
+108
View File
@@ -14,6 +14,18 @@ type HTTPInfrastructurePersistenceInterface interface {
CreateEvent(types.TenantID, *types.BusinessEvent) error
ListPendingEvents(types.TenantID, int, time.Time) ([]types.BusinessEvent, error)
MarkEventDelivered(types.TenantID, string, time.Time) error
BeginKafkaConsume(types.TenantID, *types.KafkaConsumeRecord, time.Time) (*types.KafkaConsumeRecord, bool, error)
MarkKafkaConsumeSucceeded(types.TenantID, string, string, time.Time) error
MarkKafkaConsumeFailed(types.TenantID, string, string, string, string, time.Time) (*types.KafkaConsumeRecord, error)
MarkKafkaConsumeDead(types.TenantID, *types.KafkaDeadLetter, time.Time) error
ListKafkaDeadLetters(types.TenantID, types.KafkaDeadLetterListOptions) (*types.KafkaDeadLetterPage, error)
GetKafkaDeadLetter(types.TenantID, string) (*types.KafkaDeadLetter, error)
MarkKafkaDeadLetter(types.TenantID, string, string, string, string, time.Time) error
MarkKafkaDeadLetterReplay(types.TenantID, string, string, string, time.Time) (*types.KafkaDeadLetter, error)
CreateKafkaOutbox(types.TenantID, *types.KafkaOutboxRecord) error
ListPendingKafkaOutbox(types.TenantID, int, time.Time) ([]types.KafkaOutboxRecord, error)
MarkKafkaOutboxSent(types.TenantID, string, time.Time) error
MarkKafkaOutboxFailed(types.TenantID, string, string, string, time.Time, time.Time) error
}
type httpInfrastructureMapper struct{}
@@ -79,3 +91,99 @@ func (httpInfrastructureMapper) MarkEventDelivered(tenantID types.TenantID, even
}
return infrastructure.TenantBusinessEventMarkDelivered(tenantID, eventID, now)
}
func (httpInfrastructureMapper) BeginKafkaConsume(tenantID types.TenantID, record *types.KafkaConsumeRecord, deadline time.Time) (*types.KafkaConsumeRecord, bool, error) {
infrastructure, err := tenantHTTPInfrastructureAdapter(tenantID)
if err != nil {
return nil, false, err
}
return infrastructure.TenantKafkaConsumeBegin(tenantID, record, deadline)
}
func (httpInfrastructureMapper) MarkKafkaConsumeSucceeded(tenantID types.TenantID, eventID, consumerGroup string, now time.Time) error {
infrastructure, err := tenantHTTPInfrastructureAdapter(tenantID)
if err != nil {
return err
}
return infrastructure.TenantKafkaConsumeSucceed(tenantID, eventID, consumerGroup, now)
}
func (httpInfrastructureMapper) MarkKafkaConsumeFailed(tenantID types.TenantID, eventID, consumerGroup, reason, message string, now time.Time) (*types.KafkaConsumeRecord, error) {
infrastructure, err := tenantHTTPInfrastructureAdapter(tenantID)
if err != nil {
return nil, err
}
return infrastructure.TenantKafkaConsumeFail(tenantID, eventID, consumerGroup, reason, message, now)
}
func (httpInfrastructureMapper) MarkKafkaConsumeDead(tenantID types.TenantID, dead *types.KafkaDeadLetter, now time.Time) error {
infrastructure, err := tenantHTTPInfrastructureAdapter(tenantID)
if err != nil {
return err
}
return infrastructure.TenantKafkaConsumeDead(tenantID, dead, now)
}
func (httpInfrastructureMapper) ListKafkaDeadLetters(tenantID types.TenantID, opts types.KafkaDeadLetterListOptions) (*types.KafkaDeadLetterPage, error) {
infrastructure, err := tenantHTTPInfrastructureAdapter(tenantID)
if err != nil {
return nil, err
}
return infrastructure.TenantKafkaDeadLetterList(tenantID, opts)
}
func (httpInfrastructureMapper) GetKafkaDeadLetter(tenantID types.TenantID, deadID string) (*types.KafkaDeadLetter, error) {
infrastructure, err := tenantHTTPInfrastructureAdapter(tenantID)
if err != nil {
return nil, err
}
return infrastructure.TenantKafkaDeadLetterGet(tenantID, deadID)
}
func (httpInfrastructureMapper) MarkKafkaDeadLetter(tenantID types.TenantID, deadID, status, resolvedBy, note string, now time.Time) error {
infrastructure, err := tenantHTTPInfrastructureAdapter(tenantID)
if err != nil {
return err
}
return infrastructure.TenantKafkaDeadLetterMark(tenantID, deadID, status, resolvedBy, note, now)
}
func (httpInfrastructureMapper) MarkKafkaDeadLetterReplay(tenantID types.TenantID, deadID, resolvedBy, note string, now time.Time) (*types.KafkaDeadLetter, error) {
infrastructure, err := tenantHTTPInfrastructureAdapter(tenantID)
if err != nil {
return nil, err
}
return infrastructure.TenantKafkaDeadLetterMarkReplay(tenantID, deadID, resolvedBy, note, now)
}
func (httpInfrastructureMapper) CreateKafkaOutbox(tenantID types.TenantID, record *types.KafkaOutboxRecord) error {
infrastructure, err := tenantHTTPInfrastructureAdapter(tenantID)
if err != nil {
return err
}
return infrastructure.TenantKafkaOutboxCreate(tenantID, record)
}
func (httpInfrastructureMapper) ListPendingKafkaOutbox(tenantID types.TenantID, limit int, now time.Time) ([]types.KafkaOutboxRecord, error) {
infrastructure, err := tenantHTTPInfrastructureAdapter(tenantID)
if err != nil {
return nil, err
}
return infrastructure.TenantKafkaOutboxListPending(tenantID, limit, now)
}
func (httpInfrastructureMapper) MarkKafkaOutboxSent(tenantID types.TenantID, eventID string, now time.Time) error {
infrastructure, err := tenantHTTPInfrastructureAdapter(tenantID)
if err != nil {
return err
}
return infrastructure.TenantKafkaOutboxMarkSent(tenantID, eventID, now)
}
func (httpInfrastructureMapper) MarkKafkaOutboxFailed(tenantID types.TenantID, eventID, reason, message string, nextRetryAt, now time.Time) error {
infrastructure, err := tenantHTTPInfrastructureAdapter(tenantID)
if err != nil {
return err
}
return infrastructure.TenantKafkaOutboxMarkFailed(tenantID, eventID, reason, message, nextRetryAt, now)
}
+43
View File
@@ -0,0 +1,43 @@
package store
import (
adapter "github.com/tinode/chat/server/db"
"github.com/tinode/chat/server/store/types"
)
type InternalAdminQueryPersistenceInterface interface {
Query(types.TenantID, string, string, []any, int, int) (*types.InternalAdminQueryResult, error)
QueryOne(types.TenantID, string, []any) (map[string]any, error)
}
type internalAdminQueryMapper struct{}
var InternalAdminQueries InternalAdminQueryPersistenceInterface = internalAdminQueryMapper{}
func internalAdminQueryAdapter(tenantID types.TenantID) (adapter.TenantInternalAdminQueryAdapter, error) {
business, err := tenantBusinessAdapter(tenantID)
if err != nil {
return nil, err
}
queryAdp, ok := business.(adapter.TenantInternalAdminQueryAdapter)
if !ok {
return nil, types.ErrUnsupported
}
return queryAdp, nil
}
func (internalAdminQueryMapper) Query(tenantID types.TenantID, query string, countQuery string, args []any, page, perPage int) (*types.InternalAdminQueryResult, error) {
queryAdp, err := internalAdminQueryAdapter(tenantID)
if err != nil {
return nil, err
}
return queryAdp.TenantInternalAdminQuery(tenantID, query, countQuery, args, page, perPage)
}
func (internalAdminQueryMapper) QueryOne(tenantID types.TenantID, query string, args []any) (map[string]any, error) {
queryAdp, err := internalAdminQueryAdapter(tenantID)
if err != nil {
return nil, err
}
return queryAdp.TenantInternalAdminQueryOne(tenantID, query, args)
}
+60
View File
@@ -330,6 +330,36 @@ func (mr *MockTenantsPersistenceInterfaceMockRecorder) GetByCode(code interface{
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetByCode", reflect.TypeOf((*MockTenantsPersistenceInterface)(nil).GetByCode), code)
}
// GetByID mocks base method.
func (m *MockTenantsPersistenceInterface) GetByID(id types.TenantID) (*types.Tenant, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "GetByID", id)
ret0, _ := ret[0].(*types.Tenant)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// GetByID indicates an expected call of GetByID.
func (mr *MockTenantsPersistenceInterfaceMockRecorder) GetByID(id interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetByID", reflect.TypeOf((*MockTenantsPersistenceInterface)(nil).GetByID), id)
}
// ListActive mocks base method.
func (m *MockTenantsPersistenceInterface) ListActive(limit int) ([]types.Tenant, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "ListActive", limit)
ret0, _ := ret[0].([]types.Tenant)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// ListActive indicates an expected call of ListActive.
func (mr *MockTenantsPersistenceInterfaceMockRecorder) ListActive(limit interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListActive", reflect.TypeOf((*MockTenantsPersistenceInterface)(nil).ListActive), limit)
}
// MockUsersPersistenceInterface is a mock of UsersPersistenceInterface interface.
type MockUsersPersistenceInterface struct {
ctrl *gomock.Controller
@@ -1220,6 +1250,36 @@ func (mr *MockMessagesPersistenceInterfaceMockRecorder) Save(tenantID, msg, atta
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Save", reflect.TypeOf((*MockMessagesPersistenceInterface)(nil).Save), tenantID, msg, attachmentURLs, readBySender)
}
// PreparePersist mocks base method.
func (m *MockMessagesPersistenceInterface) PreparePersist(tenantID types.TenantID, msg *types.Message, attachmentURLs []string, readBySender bool) (*types.MessageSaveResult, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "PreparePersist", tenantID, msg, attachmentURLs, readBySender)
ret0, _ := ret[0].(*types.MessageSaveResult)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// PreparePersist indicates an expected call of PreparePersist.
func (mr *MockMessagesPersistenceInterfaceMockRecorder) PreparePersist(tenantID, msg, attachmentURLs, readBySender interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PreparePersist", reflect.TypeOf((*MockMessagesPersistenceInterface)(nil).PreparePersist), tenantID, msg, attachmentURLs, readBySender)
}
// PersistPrepared mocks base method.
func (m *MockMessagesPersistenceInterface) PersistPrepared(tenantID types.TenantID, msg *types.Message, attachmentURLs []string, readBySender bool) (*types.MessageSaveResult, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "PersistPrepared", tenantID, msg, attachmentURLs, readBySender)
ret0, _ := ret[0].(*types.MessageSaveResult)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// PersistPrepared indicates an expected call of PersistPrepared.
func (mr *MockMessagesPersistenceInterfaceMockRecorder) PersistPrepared(tenantID, msg, attachmentURLs, readBySender interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PersistPrepared", reflect.TypeOf((*MockMessagesPersistenceInterface)(nil).PersistPrepared), tenantID, msg, attachmentURLs, readBySender)
}
// MockDevicePersistenceInterface is a mock of DevicePersistenceInterface interface.
type MockDevicePersistenceInterface struct {
ctrl *gomock.Controller
+115 -2
View File
@@ -151,6 +151,7 @@ const (
type TenantsPersistenceInterface interface {
GetByCode(code string) (*types.Tenant, error)
GetByID(id types.TenantID) (*types.Tenant, error)
ListActive(limit int) ([]types.Tenant, error)
}
type tenantsMapper struct{}
@@ -158,6 +159,22 @@ type tenantsMapper struct{}
// Tenants is the tenant registry access object.
var Tenants TenantsPersistenceInterface
// AdminInternalConfigProvider optionally resolves hotgo-owned configuration
// through the Admin service instead of reading the hotgo database directly.
type AdminInternalConfigProvider interface {
GetTenantByCode(code string) (*types.Tenant, error)
GetTenantByID(id types.TenantID) (*types.Tenant, error)
ListActiveTenants(limit int) ([]types.Tenant, error)
RegistrationSMSEnabled() (bool, error)
ThirdPartyConfig(group string, keys ...string) (map[string]string, error)
}
var adminInternalConfigProvider AdminInternalConfigProvider
func SetAdminInternalConfigProvider(provider AdminInternalConfigProvider) {
adminInternalConfigProvider = provider
}
// RegistrationConfigPersistenceInterface exposes global registration policy.
type RegistrationConfigPersistenceInterface interface {
SMSEnabled() (bool, error)
@@ -172,6 +189,14 @@ func (registrationConfigMapper) SMSEnabled() (bool, error) {
if enabled, ok := registrationSMSCache.get("im:register_sms_enabled"); ok {
return enabled, nil
}
if adminInternalConfigProvider != nil {
if enabled, err := adminInternalConfigProvider.RegistrationSMSEnabled(); err == nil {
registrationSMSCache.set("im:register_sms_enabled", enabled)
return enabled, nil
} else {
logs.Warn.Printf("admin internal registration config fallback failed: %v", err)
}
}
configAdp, ok := adp.(adapter.RegistrationConfigAdapter)
if !ok {
return true, types.ErrUnsupported
@@ -198,6 +223,14 @@ func (thirdPartyConfigMapper) Get(group string, keys ...string) (map[string]stri
if cached, ok := thirdPartyConfigCache.get(cacheKey); ok {
return cloneStringMap(cached), nil
}
if adminInternalConfigProvider != nil {
if result, err := adminInternalConfigProvider.ThirdPartyConfig(group, keys...); err == nil {
thirdPartyConfigCache.set(cacheKey, cloneStringMap(result))
return result, nil
} else {
logs.Warn.Printf("admin internal third-party config fallback failed: group=%s err=%v", group, err)
}
}
configAdp, ok := adp.(adapter.ThirdPartyConfigAdapter)
if !ok {
return nil, types.ErrUnsupported
@@ -261,6 +294,15 @@ func (tenantsMapper) GetByCode(code string) (*types.Tenant, error) {
if tenant, ok := tenantByCodeCache.get(code); ok {
return cloneTenant(tenant), nil
}
if adminInternalConfigProvider != nil {
if tenant, err := adminInternalConfigProvider.GetTenantByCode(code); err == nil && tenant != nil {
tenantByCodeCache.set(code, cloneTenant(tenant))
tenantByIDCache.set(strconv.FormatInt(int64(tenant.ID), 10), cloneTenant(tenant))
return tenant, nil
} else if err != nil {
logs.Warn.Printf("admin internal tenant by code fallback failed: code=%s err=%v", code, err)
}
}
tenantAdp, ok := adp.(adapter.TenantAdapter)
if !ok {
return nil, types.ErrUnsupported
@@ -282,6 +324,15 @@ func (tenantsMapper) GetByID(id types.TenantID) (*types.Tenant, error) {
if tenant, ok := tenantByIDCache.get(idKey); ok {
return cloneTenant(tenant), nil
}
if adminInternalConfigProvider != nil {
if tenant, err := adminInternalConfigProvider.GetTenantByID(id); err == nil && tenant != nil {
tenantByIDCache.set(idKey, cloneTenant(tenant))
tenantByCodeCache.set(tenant.Code, cloneTenant(tenant))
return tenant, nil
} else if err != nil {
logs.Warn.Printf("admin internal tenant by id fallback failed: tenant=%d err=%v", id, err)
}
}
tenantAdp, ok := adp.(adapter.TenantAdapter)
if !ok {
return nil, types.ErrUnsupported
@@ -294,6 +345,29 @@ func (tenantsMapper) GetByID(id types.TenantID) (*types.Tenant, error) {
return tenant, err
}
func (tenantsMapper) ListActive(limit int) ([]types.Tenant, error) {
if limit <= 0 {
limit = 1000
}
if adminInternalConfigProvider != nil {
if tenants, err := adminInternalConfigProvider.ListActiveTenants(limit); err == nil {
for i := range tenants {
tenant := tenants[i]
tenantByIDCache.set(strconv.FormatInt(int64(tenant.ID), 10), cloneTenant(&tenant))
tenantByCodeCache.set(tenant.Code, cloneTenant(&tenant))
}
return tenants, nil
} else {
logs.Warn.Printf("admin internal active tenants fallback failed: err=%v", err)
}
}
tenantAdp, ok := adp.(adapter.TenantAdapter)
if !ok {
return nil, types.ErrUnsupported
}
return tenantAdp.TenantListActive(limit)
}
func tenantAuthAdapter(tenantID types.TenantID) (adapter.TenantAuthAdapter, error) {
if !tenantID.IsValid() {
return nil, types.ErrMalformed
@@ -1267,6 +1341,8 @@ func subscriptionCacheID(topic string, user types.Uid, keepDeleted bool) string
// MessagesPersistenceInterface is an interface which defines methods for persistent storage of messages.
type MessagesPersistenceInterface interface {
Save(tenantID types.TenantID, msg *types.Message, attachmentURLs []string, readBySender bool) (*types.MessageSaveResult, error)
PreparePersist(tenantID types.TenantID, msg *types.Message, attachmentURLs []string, readBySender bool) (*types.MessageSaveResult, error)
PersistPrepared(tenantID types.TenantID, msg *types.Message, attachmentURLs []string, readBySender bool) (*types.MessageSaveResult, error)
GetOne(tenantID types.TenantID, topic string, seq int64) (*types.Message, error)
Recall(tenantID types.TenantID, topic string, seq int64, actor types.Uid, expectedVersion int, recalledAt time.Time) (*types.Message, error)
Edit(tenantID types.TenantID, topic string, seq int64, actor types.Uid, expectedVersion int, head types.KVMap, content any, reason string, editedAt time.Time) (*types.Message, error)
@@ -3063,6 +3139,17 @@ func cachedBlockContains(tenantID types.TenantID, blocker, blocked types.Uid) (b
// Save message
func (messagesMapper) Save(tenantID types.TenantID, msg *types.Message, attachmentURLs []string, readBySender bool) (*types.MessageSaveResult, error) {
saveResult, err := Messages.PreparePersist(tenantID, msg, attachmentURLs, readBySender)
if err != nil {
return nil, err
}
if saveResult != nil && saveResult.Duplicate {
return saveResult, nil
}
return Messages.PersistPrepared(tenantID, msg, attachmentURLs, readBySender)
}
func (messagesMapper) PreparePersist(tenantID types.TenantID, msg *types.Message, _ []string, _ bool) (*types.MessageSaveResult, error) {
if msg == nil || msg.TenantID != tenantID {
return nil, types.ErrMalformed
}
@@ -3126,12 +3213,29 @@ func (messagesMapper) Save(tenantID types.TenantID, msg *types.Message, attachme
}
msg.InitTimes()
msg.SetUid(Store.GetUid())
// Increment topic's or user's SeqId
// Reserve topic's or user's SeqId before optional async persistence.
err = tenantAdp.TenantTopicUpdateOnMessage(tenantID, msg.Topic, msg)
if err != nil {
return nil, err
}
return &types.MessageSaveResult{SeqId: msg.SeqId}, nil
}
func (messagesMapper) PersistPrepared(tenantID types.TenantID, msg *types.Message, attachmentURLs []string, readBySender bool) (*types.MessageSaveResult, error) {
if msg == nil || msg.TenantID != tenantID || msg.Topic == "" || msg.SeqId <= 0 {
return nil, types.ErrMalformed
}
tenantAdp, err := tenantBusinessAdapter(tenantID)
if err != nil {
return nil, err
}
clientMsgID, err := normalizeClientMsgID(msg.Head)
if err != nil {
return nil, err
}
msg.ClientMsgID = clientMsgID
fromUid := types.ParseUid(msg.From)
isGroupMessage := types.GetTopicCat(msg.Topic) == types.TopicCatGrp
if isGroupMessage {
err = tenantAdp.TenantGroupMessageSave(tenantID, msg)
} else {
@@ -3220,6 +3324,14 @@ func (messagesMapper) Save(tenantID types.TenantID, msg *types.Message, attachme
return &types.MessageSaveResult{SeqId: msg.SeqId, MarkedReadBySender: markedReadBySender}, nil
}
func PrepareMessagePersist(tenantID types.TenantID, msg *types.Message, attachmentURLs []string, readBySender bool) (*types.MessageSaveResult, error) {
return Messages.PreparePersist(tenantID, msg, attachmentURLs, readBySender)
}
func PersistPreparedMessage(tenantID types.TenantID, msg *types.Message, attachmentURLs []string, readBySender bool) (*types.MessageSaveResult, error) {
return Messages.PersistPrepared(tenantID, msg, attachmentURLs, readBySender)
}
func retryTenantMessageSaveAfterSeqConflict(tenantAdp adapter.TenantBusinessAdapter, tenantID types.TenantID, msg *types.Message) error {
latest, err := tenantAdp.TenantMessageGetAll(tenantID, msg.Topic, types.ZeroUid, &types.QueryOpt{Before: 1 << 62, Limit: 1})
if err != nil {
@@ -3821,7 +3933,7 @@ func (fileMapper) LinkAttachments(tenantID types.TenantID, topic string, msgId t
if len(fids) > 0 {
userId := types.ZeroUid
if types.GetTopicCat(topic) == types.TopicCatMe {
if topic != "" && types.GetTopicCat(topic) == types.TopicCatMe {
userId = types.ParseUserId(topic)
topic = ""
}
@@ -4353,6 +4465,7 @@ func init() {
Tenants = tenantsMapper{}
RegistrationConfig = registrationConfigMapper{}
TenantEntry = tenantEntryMapper{}
InternalAdminQueries = internalAdminQueryMapper{}
Users = usersMapper{}
Topics = topicsMapper{}
Subs = subsMapper{}
+89
View File
@@ -12,6 +12,21 @@ const (
BusinessEventPending = "pending"
BusinessEventDelivered = "delivered"
KafkaConsumeProcessing = "processing"
KafkaConsumeSucceeded = "succeeded"
KafkaConsumeFailed = "failed"
KafkaConsumeDead = "dead"
KafkaDeadStatusDead = "dead"
KafkaDeadStatusPendingReplay = "pending_replay"
KafkaDeadStatusReplayed = "replayed"
KafkaDeadStatusIgnored = "ignored"
KafkaDeadStatusResolved = "resolved"
KafkaOutboxPending = "pending"
KafkaOutboxSent = "sent"
KafkaOutboxFailed = "failed"
)
// HTTPIdempotencyRecord stores the first response for one authenticated write.
@@ -46,6 +61,80 @@ type BusinessEvent struct {
UpdatedAt time.Time
}
// KafkaConsumeRecord stores idempotent consumer state for one event and group.
type KafkaConsumeRecord struct {
EventID string
ConsumerGroup string
SourceTopic string
EventType string
BizType string
BizID string
Status string
RetryCount int
LastReason string
LastError string
ProcessingDeadline *time.Time
SucceededAt *time.Time
DeadAt *time.Time
CreatedAt time.Time
UpdatedAt time.Time
}
// KafkaDeadLetter stores a poison Kafka event for manual handling.
type KafkaDeadLetter struct {
ID int64
DeadID string
EventID string
ParentEventID string
SourceTopic string
ConsumerGroup string
EventType string
BizType string
BizID string
Envelope []byte
RetryCount int
LastReason string
LastError string
Status string
ReplayCount int
CreatedAt time.Time
UpdatedAt time.Time
}
type KafkaDeadLetterListOptions struct {
Status string
BizType string
BizID string
EventType string
Cursor string
Limit int
}
type KafkaDeadLetterPage struct {
Items []KafkaDeadLetter
NextCursor string
}
// KafkaOutboxRecord stores a Kafka event waiting for reliable publication.
type KafkaOutboxRecord struct {
ID int64
EventID string
Topic string
EventType string
BizType string
BizID string
Key string
Envelope []byte
Status string
Attempts int
NextRetryAt *time.Time
SentAt *time.Time
LastReason string
LastError string
CreatedAt time.Time
UpdatedAt time.Time
}
// StableEventID derives an idempotent event identifier from domain identity.
func StableEventID(parts ...any) string {
hash := sha256.New()
@@ -0,0 +1,8 @@
package types
type InternalAdminQueryResult struct {
Items []map[string]any
Total int64
Page int
PerPage int
}
+47
View File
@@ -106,6 +106,53 @@
}
},
"kafka": {
"enabled": false,
"brokers": [],
"client_id": "im-server",
"consumer_group": "im-server",
"dial_timeout_ms": 3000,
"write_timeout_ms": 3000,
"read_timeout_ms": 3000,
"required_acks": 1,
"allow_auto_topic_creation": false,
"tls_enabled": false,
"max_retry_count": 5,
"retry_backoff_ms": [10000, 30000, 60000, 300000, 900000],
"dead_letter_retention_seconds": 7776000,
"topics": {
"admin_command": "im.admin.command",
"admin_command_result": "im.admin.command.result",
"config_event": "im.config.event",
"business_event": "im.business.event",
"message_persist": "im.message.persist",
"message_push": "im.message.push",
"file_security": "im.file.security",
"file_lifecycle": "im.file.lifecycle",
"workbench_event": "im.workbench.event",
"cache_invalidate": "im.cache.invalidate",
"audit_event": "im.audit.event",
"dead_letter": "im.dead.letter"
}
},
"internal_api": {
"enabled": false,
"time_window_seconds": 300,
"services": {
"hotgo-admin": {
"secret": "",
"allow": ["admin_query"]
}
}
},
"admin_internal_api": {
"enabled": false,
"base_url": "",
"service_name": "im-server",
"secret": "",
"timeout_ms": 3000
},
"acc_validation": {
"email": {
"add_to_tags": false,
+2 -1
View File
@@ -9,6 +9,7 @@
package main
import (
"context"
"errors"
"sort"
"strings"
@@ -1104,7 +1105,7 @@ func (t *Topic) saveAndBroadcastMessage(msg *ClientComMessage, asUid types.Uid,
}
}
saveResult, err := store.Messages.Save(t.tenantID,
saveResult, err := saveMessageMaybeKafka(context.Background(), t.tenantID,
&types.Message{
ObjHeader: types.ObjHeader{CreatedAt: msg.Timestamp},
TenantID: t.tenantID,